@leia-org/luke-server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/providers/gemini.d.ts +8 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +258 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/openai.d.ts +8 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +195 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/server/auth.d.ts +4 -0
- package/dist/server/auth.d.ts.map +1 -0
- package/dist/server/auth.js +50 -0
- package/dist/server/auth.js.map +1 -0
- package/dist/server/ws-server.d.ts +9 -0
- package/dist/server/ws-server.d.ts.map +1 -0
- package/dist/server/ws-server.js +264 -0
- package/dist/server/ws-server.js.map +1 -0
- package/dist/types.d.ts +130 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +4 -0
- package/dist/types.js.map +1 -0
- package/package.json +40 -0
package/README.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# @leia-org/luke-server
|
|
2
|
+
|
|
3
|
+
The server-side component of the Luke library, responsible for managing WebSocket connections, authentication, and orchestrating communication with AI providers (OpenAI Realtime, Gemini Live).
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Unified Interface**: Abstract away differences between OpenAI and Gemini.
|
|
8
|
+
- **WebSocket Server**: Handles real-time audio and text streaming.
|
|
9
|
+
- **Authentication**: Built-in support for JWT and custom validation.
|
|
10
|
+
- **Session Management**: persistent sessions and database integration hooks.
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pnpm add @leia-org/luke-server
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Usage
|
|
19
|
+
|
|
20
|
+
```typescript
|
|
21
|
+
import { createLukeServer, openai, gemini } from '@leia-org/luke-server';
|
|
22
|
+
|
|
23
|
+
const server = createLukeServer({
|
|
24
|
+
providers: [
|
|
25
|
+
openai({ apiKey: process.env.OPENAI_API_KEY }),
|
|
26
|
+
gemini({ apiKey: process.env.GEMINI_API_KEY }),
|
|
27
|
+
],
|
|
28
|
+
// ... configuration
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
server.listen(3001, () => {
|
|
32
|
+
console.log('Luke server running on port 3001');
|
|
33
|
+
});
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Configuration
|
|
37
|
+
|
|
38
|
+
The `createLukeServer` function accepts a configuration object with the following properties:
|
|
39
|
+
|
|
40
|
+
| Option | Type | Required | Description |
|
|
41
|
+
|--------|------|----------|-------------|
|
|
42
|
+
| `providers` | `LukeProvider[]` | Yes | Array of initialized provider instances (e.g., `openai({...})`, `gemini({...})`). |
|
|
43
|
+
| `auth` | `AuthConfig` | Yes | Configuration for authentication. |
|
|
44
|
+
| `auth.jwt` | `JwtConfig` | No | JWT verification settings (`secret`, `algorithms`). |
|
|
45
|
+
| `auth.validate` | `Function` | Yes | Callback to validate decoded token and return user object. |
|
|
46
|
+
| `session` | `SessionConfig` | No | Hooks for session lifecycle management. |
|
|
47
|
+
| `session.create` | `Function` | No | Called when a new session is established. |
|
|
48
|
+
| `session.onEnd` | `Function` | No | Called when a session ends (disconnect, error, timeout). |
|
|
49
|
+
| `config` | `ProviderSessionConfig` | No | Default configuration for provider sessions. |
|
|
50
|
+
| `config.systemInstruction` | `string` | No | System prompt for the AI model. |
|
|
51
|
+
| `config.tools` | `ToolDefinition[]` | No | Tools available to the model. |
|
|
52
|
+
| `onConnect` | `Function` | No | Callback when a client successfully connects. |
|
|
53
|
+
| `onDisconnect` | `Function` | No | Callback when a client disconnects. |
|
|
54
|
+
| `onTranscription` | `Function` | No | Callback for real-time transcription events. |
|
|
55
|
+
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { createLukeServer } from './server/ws-server.js';
|
|
2
|
+
export type { LukeServerInstance } from './server/ws-server.js';
|
|
3
|
+
export { openai } from './providers/openai.js';
|
|
4
|
+
export { gemini } from './providers/gemini.js';
|
|
5
|
+
export type { LukeProvider, ProviderConnection, ProviderSessionConfig, VoiceConfig, LukeServerConfig, AuthConfig, JwtConfig, SessionConfig, LukeSession, Transcription, ToolDefinition, ClientMessage, ServerMessage, HandshakeMessage, } from './types.js';
|
|
6
|
+
export { z } from 'zod';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,YAAY,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,YAAY,EAER,YAAY,EACZ,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,EAGX,gBAAgB,EAChB,UAAU,EACV,SAAS,EACT,aAAa,EAGb,WAAW,EACX,aAAa,EAGb,cAAc,EAGd,aAAa,EACb,aAAa,EACb,gBAAgB,GACnB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// @luke/server - Unified Realtime AI Server
|
|
2
|
+
// Main entry point and public exports
|
|
3
|
+
export { createLukeServer } from './server/ws-server.js';
|
|
4
|
+
export { openai } from './providers/openai.js';
|
|
5
|
+
export { gemini } from './providers/gemini.js';
|
|
6
|
+
// Re-export zod for tool parameter definitions
|
|
7
|
+
export { z } from 'zod';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,sCAAsC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AA4B/C,+CAA+C;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../src/providers/gemini.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACR,YAAY,EAMf,MAAM,aAAa,CAAC;AAErB,UAAU,qBAAqB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAYD,wBAAgB,MAAM,CAAC,OAAO,EAAE,qBAAqB,GAAG,YAAY,CAanE"}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
// Gemini Live Provider
|
|
2
|
+
// Connects to Google's Gemini Live API via WebSocket
|
|
3
|
+
import WebSocket from 'ws';
|
|
4
|
+
// Gemini uses speechConfig for voice selection
|
|
5
|
+
const GEMINI_VOICES = [
|
|
6
|
+
{ id: 'Puck', name: 'Puck' },
|
|
7
|
+
{ id: 'Charon', name: 'Charon' },
|
|
8
|
+
{ id: 'Kore', name: 'Kore' },
|
|
9
|
+
{ id: 'Fenrir', name: 'Fenrir' },
|
|
10
|
+
{ id: 'Aoede', name: 'Aoede' },
|
|
11
|
+
];
|
|
12
|
+
// Creates a Gemini Live provider instance
|
|
13
|
+
export function gemini(options) {
|
|
14
|
+
const model = options.model ?? 'gemini-2.5-flash-native-audio-preview-12-2025';
|
|
15
|
+
return {
|
|
16
|
+
id: 'gemini',
|
|
17
|
+
name: 'gemini',
|
|
18
|
+
sampleRate: 16000, // Input sample rate
|
|
19
|
+
voices: GEMINI_VOICES,
|
|
20
|
+
async connect(config) {
|
|
21
|
+
return createGeminiConnection(options.apiKey, model, config);
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
// Manages the WebSocket connection to Gemini Live API
|
|
26
|
+
async function createGeminiConnection(apiKey, model, config) {
|
|
27
|
+
// Gemini Live API WebSocket URL (v1beta per documentation)
|
|
28
|
+
const baseUrl = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
|
|
29
|
+
const url = `${baseUrl}?key=${apiKey}`;
|
|
30
|
+
const ws = new WebSocket(url);
|
|
31
|
+
// Event handlers
|
|
32
|
+
let onAudioHandler = null;
|
|
33
|
+
let onTranscriptionHandler = null;
|
|
34
|
+
let onTurnCompleteHandler = null;
|
|
35
|
+
let onInterruptedHandler = null;
|
|
36
|
+
let onErrorHandler = null;
|
|
37
|
+
// Wait for connection and send setup
|
|
38
|
+
await new Promise((resolve, reject) => {
|
|
39
|
+
const timeout = setTimeout(() => reject(new Error('Connection timeout')), 15000);
|
|
40
|
+
ws.on('open', () => {
|
|
41
|
+
// Build setup message per API spec
|
|
42
|
+
const setup = {
|
|
43
|
+
model: `models/${model}`,
|
|
44
|
+
generationConfig: {
|
|
45
|
+
responseModalities: ['AUDIO'],
|
|
46
|
+
speechConfig: {
|
|
47
|
+
voiceConfig: {
|
|
48
|
+
prebuiltVoiceConfig: {
|
|
49
|
+
voiceName: config.voice ?? 'Puck',
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
// Add system instruction if provided
|
|
56
|
+
if (config.systemInstruction) {
|
|
57
|
+
setup.systemInstruction = {
|
|
58
|
+
parts: [{ text: config.systemInstruction }],
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
// Add transcription config
|
|
62
|
+
if (config.transcription?.input) {
|
|
63
|
+
setup.inputAudioTranscription = {};
|
|
64
|
+
}
|
|
65
|
+
if (config.transcription?.output) {
|
|
66
|
+
setup.outputAudioTranscription = {};
|
|
67
|
+
}
|
|
68
|
+
ws.send(JSON.stringify({ setup }));
|
|
69
|
+
});
|
|
70
|
+
ws.on('message', (data) => {
|
|
71
|
+
try {
|
|
72
|
+
const message = JSON.parse(data.toString());
|
|
73
|
+
// Wait for setupComplete before resolving
|
|
74
|
+
if (message.setupComplete !== undefined) {
|
|
75
|
+
clearTimeout(timeout);
|
|
76
|
+
resolve();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
// Ignore parse errors during setup
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
ws.on('error', (err) => {
|
|
84
|
+
clearTimeout(timeout);
|
|
85
|
+
reject(err);
|
|
86
|
+
});
|
|
87
|
+
ws.on('close', (code, reason) => {
|
|
88
|
+
clearTimeout(timeout);
|
|
89
|
+
reject(new Error(`WebSocket closed: ${code}`));
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
// Handle incoming messages after setup
|
|
93
|
+
ws.on('message', (data) => {
|
|
94
|
+
try {
|
|
95
|
+
const message = JSON.parse(data.toString());
|
|
96
|
+
handleGeminiMessage(message);
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
onErrorHandler?.(err instanceof Error ? err : new Error(String(err)));
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
// Transcription buffers and tracking
|
|
103
|
+
let inputTranscriptBuffer = '';
|
|
104
|
+
let outputTranscriptBuffer = '';
|
|
105
|
+
let lastSentInputText = '';
|
|
106
|
+
let lastSentOutputText = '';
|
|
107
|
+
function handleGeminiMessage(message) {
|
|
108
|
+
const serverContent = message.serverContent;
|
|
109
|
+
if (serverContent) {
|
|
110
|
+
// Handle audio from model turn
|
|
111
|
+
const modelTurn = serverContent.modelTurn;
|
|
112
|
+
if (modelTurn?.parts) {
|
|
113
|
+
const parts = modelTurn.parts;
|
|
114
|
+
for (const part of parts) {
|
|
115
|
+
if (part.inlineData) {
|
|
116
|
+
const inlineData = part.inlineData;
|
|
117
|
+
const audioB64 = inlineData.data;
|
|
118
|
+
if (audioB64) {
|
|
119
|
+
const audioBytes = Buffer.from(audioB64, 'base64');
|
|
120
|
+
onAudioHandler?.(new Uint8Array(audioBytes));
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// Handle input transcription (user speech)
|
|
126
|
+
const inputTranscription = serverContent.inputTranscription;
|
|
127
|
+
if (inputTranscription?.text) {
|
|
128
|
+
inputTranscriptBuffer += inputTranscription.text;
|
|
129
|
+
// Send as partial
|
|
130
|
+
onTranscriptionHandler?.({
|
|
131
|
+
role: 'user',
|
|
132
|
+
text: inputTranscriptBuffer,
|
|
133
|
+
final: false,
|
|
134
|
+
});
|
|
135
|
+
lastSentInputText = inputTranscriptBuffer;
|
|
136
|
+
}
|
|
137
|
+
// Handle output transcription (assistant speech)
|
|
138
|
+
const outputTranscription = serverContent.outputTranscription;
|
|
139
|
+
if (outputTranscription?.text) {
|
|
140
|
+
// If we were accumulating user input and assistant starts speaking,
|
|
141
|
+
// finalize the user message now (before adding assistant text)
|
|
142
|
+
if (inputTranscriptBuffer && inputTranscriptBuffer !== '') {
|
|
143
|
+
onTranscriptionHandler?.({
|
|
144
|
+
role: 'user',
|
|
145
|
+
text: inputTranscriptBuffer,
|
|
146
|
+
final: true,
|
|
147
|
+
});
|
|
148
|
+
inputTranscriptBuffer = ''; // Clear so we don't re-send
|
|
149
|
+
}
|
|
150
|
+
outputTranscriptBuffer += outputTranscription.text;
|
|
151
|
+
onTranscriptionHandler?.({
|
|
152
|
+
role: 'assistant',
|
|
153
|
+
text: outputTranscriptBuffer,
|
|
154
|
+
final: false,
|
|
155
|
+
});
|
|
156
|
+
lastSentOutputText = outputTranscriptBuffer;
|
|
157
|
+
}
|
|
158
|
+
// Check for turn completion
|
|
159
|
+
if (serverContent.turnComplete) {
|
|
160
|
+
// Finalize any remaining transcriptions that weren't already finalized
|
|
161
|
+
if (inputTranscriptBuffer && inputTranscriptBuffer !== '') {
|
|
162
|
+
onTranscriptionHandler?.({
|
|
163
|
+
role: 'user',
|
|
164
|
+
text: inputTranscriptBuffer,
|
|
165
|
+
final: true,
|
|
166
|
+
});
|
|
167
|
+
inputTranscriptBuffer = '';
|
|
168
|
+
}
|
|
169
|
+
if (outputTranscriptBuffer && outputTranscriptBuffer !== '') {
|
|
170
|
+
onTranscriptionHandler?.({
|
|
171
|
+
role: 'assistant',
|
|
172
|
+
text: outputTranscriptBuffer,
|
|
173
|
+
final: true,
|
|
174
|
+
});
|
|
175
|
+
outputTranscriptBuffer = '';
|
|
176
|
+
}
|
|
177
|
+
onTurnCompleteHandler?.();
|
|
178
|
+
}
|
|
179
|
+
// Check for interruption
|
|
180
|
+
if (serverContent.interrupted) {
|
|
181
|
+
// Clear buffers on interruption
|
|
182
|
+
inputTranscriptBuffer = '';
|
|
183
|
+
outputTranscriptBuffer = '';
|
|
184
|
+
onInterruptedHandler?.();
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
// Handle Go Away (session ending soon)
|
|
188
|
+
if (message.goAway) {
|
|
189
|
+
onErrorHandler?.(new Error('Server requested disconnect'));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
ws.on('close', () => {
|
|
193
|
+
onErrorHandler?.(new Error('WebSocket connection closed'));
|
|
194
|
+
});
|
|
195
|
+
return {
|
|
196
|
+
send(message) {
|
|
197
|
+
if (ws.readyState !== WebSocket.OPEN)
|
|
198
|
+
return;
|
|
199
|
+
if (message.type === 'audio') {
|
|
200
|
+
// Send audio using the new format per documentation
|
|
201
|
+
const audioB64 = Buffer.from(message.data).toString('base64');
|
|
202
|
+
ws.send(JSON.stringify({
|
|
203
|
+
realtimeInput: {
|
|
204
|
+
audio: {
|
|
205
|
+
data: audioB64,
|
|
206
|
+
mimeType: 'audio/pcm;rate=16000',
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
}));
|
|
210
|
+
}
|
|
211
|
+
else if (message.type === 'text') {
|
|
212
|
+
// Send text as client content
|
|
213
|
+
ws.send(JSON.stringify({
|
|
214
|
+
clientContent: {
|
|
215
|
+
turns: [
|
|
216
|
+
{
|
|
217
|
+
role: 'user',
|
|
218
|
+
parts: [{ text: message.content }],
|
|
219
|
+
},
|
|
220
|
+
],
|
|
221
|
+
turnComplete: true,
|
|
222
|
+
},
|
|
223
|
+
}));
|
|
224
|
+
}
|
|
225
|
+
},
|
|
226
|
+
onAudio(handler) {
|
|
227
|
+
onAudioHandler = handler;
|
|
228
|
+
},
|
|
229
|
+
onTranscription(handler) {
|
|
230
|
+
onTranscriptionHandler = handler;
|
|
231
|
+
},
|
|
232
|
+
onTurnComplete(handler) {
|
|
233
|
+
onTurnCompleteHandler = handler;
|
|
234
|
+
},
|
|
235
|
+
onInterrupted(handler) {
|
|
236
|
+
onInterruptedHandler = handler;
|
|
237
|
+
},
|
|
238
|
+
onError(handler) {
|
|
239
|
+
onErrorHandler = handler;
|
|
240
|
+
},
|
|
241
|
+
interrupt() {
|
|
242
|
+
// Signal end of audio stream
|
|
243
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
244
|
+
ws.send(JSON.stringify({
|
|
245
|
+
realtimeInput: {
|
|
246
|
+
audioStreamEnd: true,
|
|
247
|
+
},
|
|
248
|
+
}));
|
|
249
|
+
}
|
|
250
|
+
},
|
|
251
|
+
async disconnect() {
|
|
252
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
253
|
+
ws.close();
|
|
254
|
+
}
|
|
255
|
+
},
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
//# sourceMappingURL=gemini.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../src/providers/gemini.ts"],"names":[],"mappings":"AAAA,uBAAuB;AACvB,qDAAqD;AAErD,OAAO,SAAS,MAAM,IAAI,CAAC;AAe3B,+CAA+C;AAC/C,MAAM,aAAa,GAAkB;IACjC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC5B,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC5B,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE;CACjC,CAAC;AAEF,0CAA0C;AAC1C,MAAM,UAAU,MAAM,CAAC,OAA8B;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,+CAA+C,CAAC;IAE/E,OAAO;QACH,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK,EAAE,oBAAoB;QACvC,MAAM,EAAE,aAAa;QAErB,KAAK,CAAC,OAAO,CAAC,MAA6B;YACvC,OAAO,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACjE,CAAC;KACJ,CAAC;AACN,CAAC;AAED,sDAAsD;AACtD,KAAK,UAAU,sBAAsB,CACjC,MAAc,EACd,KAAa,EACb,MAA6B;IAE7B,2DAA2D;IAC3D,MAAM,OAAO,GAAG,sHAAsH,CAAC;IACvI,MAAM,GAAG,GAAG,GAAG,OAAO,QAAQ,MAAM,EAAE,CAAC;IAEvC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;IAE9B,iBAAiB;IACjB,IAAI,cAAc,GAAyC,IAAI,CAAC;IAChE,IAAI,sBAAsB,GAAwC,IAAI,CAAC;IACvE,IAAI,qBAAqB,GAAwB,IAAI,CAAC;IACtD,IAAI,oBAAoB,GAAwB,IAAI,CAAC;IACrD,IAAI,cAAc,GAAoC,IAAI,CAAC;IAE3D,qCAAqC;IACrC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEjF,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YAEf,mCAAmC;YACnC,MAAM,KAAK,GAA4B;gBACnC,KAAK,EAAE,UAAU,KAAK,EAAE;gBACxB,gBAAgB,EAAE;oBACd,kBAAkB,EAAE,CAAC,OAAO,CAAC;oBAC7B,YAAY,EAAE;wBACV,WAAW,EAAE;4BACT,mBAAmB,EAAE;gCACjB,SAAS,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM;6BACpC;yBACJ;qBACJ;iBACJ;aACJ,CAAC;YAEF,qCAAqC;YACrC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,KAAK,CAAC,iBAAiB,GAAG;oBACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC;iBAC9C,CAAC;YACN,CAAC;YAED,2BAA2B;YAC3B,IAAI,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC;gBAC9B,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC;YACvC,CAAC;YACD,IAAI,MAAM,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;gBAC/B,KAAK,CAAC,wBAAwB,GAAG,EAAE,CAAC;YACxC,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;YAC9B,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAE5C,0CAA0C;gBAC1C,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;oBACtC,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,mCAAmC;YACvC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACnB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YAC5B,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,uCAAuC;IACvC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;QAC9B,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC5C,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,cAAc,EAAE,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,qCAAqC;IACrC,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,sBAAsB,GAAG,EAAE,CAAC;IAChC,IAAI,iBAAiB,GAAG,EAAE,CAAC;IAC3B,IAAI,kBAAkB,GAAG,EAAE,CAAC;IAE5B,SAAS,mBAAmB,CAAC,OAAgC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAoD,CAAC;QAEnF,IAAI,aAAa,EAAE,CAAC;YAChB,+BAA+B;YAC/B,MAAM,SAAS,GAAG,aAAa,CAAC,SAAgD,CAAC;YACjF,IAAI,SAAS,EAAE,KAAK,EAAE,CAAC;gBACnB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAuC,CAAC;gBAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACvB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBAClB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAqC,CAAC;wBAC9D,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAc,CAAC;wBAC3C,IAAI,QAAQ,EAAE,CAAC;4BACX,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;4BACnD,cAAc,EAAE,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;wBACjD,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;YAED,2CAA2C;YAC3C,MAAM,kBAAkB,GAAG,aAAa,CAAC,kBAAyD,CAAC;YACnG,IAAI,kBAAkB,EAAE,IAAI,EAAE,CAAC;gBAC3B,qBAAqB,IAAI,kBAAkB,CAAC,IAAc,CAAC;gBAC3D,kBAAkB;gBAClB,sBAAsB,EAAE,CAAC;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,KAAK;iBACf,CAAC,CAAC;gBACH,iBAAiB,GAAG,qBAAqB,CAAC;YAC9C,CAAC;YAED,iDAAiD;YACjD,MAAM,mBAAmB,GAAG,aAAa,CAAC,mBAA0D,CAAC;YACrG,IAAI,mBAAmB,EAAE,IAAI,EAAE,CAAC;gBAC5B,oEAAoE;gBACpE,+DAA+D;gBAC/D,IAAI,qBAAqB,IAAI,qBAAqB,KAAK,EAAE,EAAE,CAAC;oBACxD,sBAAsB,EAAE,CAAC;wBACrB,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,qBAAqB;wBAC3B,KAAK,EAAE,IAAI;qBACd,CAAC,CAAC;oBACH,qBAAqB,GAAG,EAAE,CAAC,CAAC,4BAA4B;gBAC5D,CAAC;gBAED,sBAAsB,IAAI,mBAAmB,CAAC,IAAc,CAAC;gBAC7D,sBAAsB,EAAE,CAAC;oBACrB,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,sBAAsB;oBAC5B,KAAK,EAAE,KAAK;iBACf,CAAC,CAAC;gBACH,kBAAkB,GAAG,sBAAsB,CAAC;YAChD,CAAC;YAED,4BAA4B;YAC5B,IAAI,aAAa,CAAC,YAAY,EAAE,CAAC;gBAC7B,uEAAuE;gBACvE,IAAI,qBAAqB,IAAI,qBAAqB,KAAK,EAAE,EAAE,CAAC;oBACxD,sBAAsB,EAAE,CAAC;wBACrB,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,qBAAqB;wBAC3B,KAAK,EAAE,IAAI;qBACd,CAAC,CAAC;oBACH,qBAAqB,GAAG,EAAE,CAAC;gBAC/B,CAAC;gBACD,IAAI,sBAAsB,IAAI,sBAAsB,KAAK,EAAE,EAAE,CAAC;oBAC1D,sBAAsB,EAAE,CAAC;wBACrB,IAAI,EAAE,WAAW;wBACjB,IAAI,EAAE,sBAAsB;wBAC5B,KAAK,EAAE,IAAI;qBACd,CAAC,CAAC;oBACH,sBAAsB,GAAG,EAAE,CAAC;gBAChC,CAAC;gBACD,qBAAqB,EAAE,EAAE,CAAC;YAC9B,CAAC;YAED,yBAAyB;YACzB,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC;gBAC5B,gCAAgC;gBAChC,qBAAqB,GAAG,EAAE,CAAC;gBAC3B,sBAAsB,GAAG,EAAE,CAAC;gBAC5B,oBAAoB,EAAE,EAAE,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,uCAAuC;QACvC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACjB,cAAc,EAAE,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,cAAc,EAAE,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,OAAO;QACH,IAAI,CAAC,OAAwB;YACzB,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;gBAAE,OAAO;YAE7C,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC3B,oDAAoD;gBACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAC9D,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACnB,aAAa,EAAE;wBACX,KAAK,EAAE;4BACH,IAAI,EAAE,QAAQ;4BACd,QAAQ,EAAE,sBAAsB;yBACnC;qBACJ;iBACJ,CAAC,CAAC,CAAC;YACR,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACjC,8BAA8B;gBAC9B,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACnB,aAAa,EAAE;wBACX,KAAK,EAAE;4BACH;gCACI,IAAI,EAAE,MAAM;gCACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;6BACrC;yBACJ;wBACD,YAAY,EAAE,IAAI;qBACrB;iBACJ,CAAC,CAAC,CAAC;YACR,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO;YACX,cAAc,GAAG,OAAO,CAAC;QAC7B,CAAC;QAED,eAAe,CAAC,OAAO;YACnB,sBAAsB,GAAG,OAAO,CAAC;QACrC,CAAC;QAED,cAAc,CAAC,OAAO;YAClB,qBAAqB,GAAG,OAAO,CAAC;QACpC,CAAC;QAED,aAAa,CAAC,OAAO;YACjB,oBAAoB,GAAG,OAAO,CAAC;QACnC,CAAC;QAED,OAAO,CAAC,OAAO;YACX,cAAc,GAAG,OAAO,CAAC;QAC7B,CAAC;QAED,SAAS;YACL,6BAA6B;YAC7B,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACnB,aAAa,EAAE;wBACX,cAAc,EAAE,IAAI;qBACvB;iBACJ,CAAC,CAAC,CAAC;YACR,CAAC;QACL,CAAC;QAED,KAAK,CAAC,UAAU;YACZ,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,EAAE,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACL,CAAC;KACJ,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACR,YAAY,EAMf,MAAM,aAAa,CAAC;AAErB,UAAU,qBAAqB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAaD,wBAAgB,MAAM,CAAC,OAAO,EAAE,qBAAqB,GAAG,YAAY,CAanE"}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// OpenAI Realtime Provider
|
|
2
|
+
// Connects to OpenAI's Realtime API via WebSocket
|
|
3
|
+
import WebSocket from 'ws';
|
|
4
|
+
// Available voices for OpenAI Realtime
|
|
5
|
+
const OPENAI_VOICES = [
|
|
6
|
+
{ id: 'alloy', name: 'Alloy' },
|
|
7
|
+
{ id: 'echo', name: 'Echo' },
|
|
8
|
+
{ id: 'fable', name: 'Fable' },
|
|
9
|
+
{ id: 'onyx', name: 'Onyx' },
|
|
10
|
+
{ id: 'nova', name: 'Nova' },
|
|
11
|
+
{ id: 'shimmer', name: 'Shimmer' },
|
|
12
|
+
];
|
|
13
|
+
// Creates an OpenAI Realtime provider instance
|
|
14
|
+
export function openai(options) {
|
|
15
|
+
const model = options.model ?? 'gpt-realtime-2025-08-28';
|
|
16
|
+
return {
|
|
17
|
+
id: 'openai',
|
|
18
|
+
name: 'openai',
|
|
19
|
+
sampleRate: 24000,
|
|
20
|
+
voices: OPENAI_VOICES,
|
|
21
|
+
async connect(config) {
|
|
22
|
+
return createOpenAIConnection(options.apiKey, model, config);
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
// Manages the WebSocket connection to OpenAI Realtime API
|
|
27
|
+
async function createOpenAIConnection(apiKey, model, config) {
|
|
28
|
+
const url = `wss://api.openai.com/v1/realtime?model=${model}`;
|
|
29
|
+
const ws = new WebSocket(url, {
|
|
30
|
+
headers: {
|
|
31
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
32
|
+
'OpenAI-Beta': 'realtime=v1',
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
// Event handlers set by the caller
|
|
36
|
+
let onAudioHandler = null;
|
|
37
|
+
let onTranscriptionHandler = null;
|
|
38
|
+
let onTurnCompleteHandler = null;
|
|
39
|
+
let onInterruptedHandler = null;
|
|
40
|
+
let onErrorHandler = null;
|
|
41
|
+
// Wait for connection to open
|
|
42
|
+
await new Promise((resolve, reject) => {
|
|
43
|
+
ws.on('open', () => {
|
|
44
|
+
// Send session configuration
|
|
45
|
+
const sessionConfig = {
|
|
46
|
+
modalities: ['audio', 'text'],
|
|
47
|
+
voice: config.voice ?? 'alloy',
|
|
48
|
+
input_audio_format: 'pcm16',
|
|
49
|
+
output_audio_format: 'pcm16',
|
|
50
|
+
turn_detection: { type: 'server_vad' },
|
|
51
|
+
};
|
|
52
|
+
if (config.systemInstruction) {
|
|
53
|
+
sessionConfig.instructions = config.systemInstruction;
|
|
54
|
+
}
|
|
55
|
+
if (config.transcription?.input) {
|
|
56
|
+
sessionConfig.input_audio_transcription = { model: 'whisper-1' };
|
|
57
|
+
}
|
|
58
|
+
// Map tools to OpenAI format
|
|
59
|
+
if (config.tools && config.tools.length > 0) {
|
|
60
|
+
sessionConfig.tools = config.tools.map((tool) => ({
|
|
61
|
+
type: 'function',
|
|
62
|
+
name: tool.name,
|
|
63
|
+
description: tool.description,
|
|
64
|
+
parameters: tool.parameters,
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
ws.send(JSON.stringify({
|
|
68
|
+
type: 'session.update',
|
|
69
|
+
session: sessionConfig,
|
|
70
|
+
}));
|
|
71
|
+
resolve();
|
|
72
|
+
});
|
|
73
|
+
ws.on('error', reject);
|
|
74
|
+
});
|
|
75
|
+
// Handle incoming messages from OpenAI
|
|
76
|
+
ws.on('message', (data) => {
|
|
77
|
+
try {
|
|
78
|
+
const message = JSON.parse(data.toString());
|
|
79
|
+
handleOpenAIMessage(message);
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
onErrorHandler?.(err instanceof Error ? err : new Error(String(err)));
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
function handleOpenAIMessage(message) {
|
|
86
|
+
switch (message.type) {
|
|
87
|
+
case 'response.audio.delta': {
|
|
88
|
+
// Audio chunk received
|
|
89
|
+
const audioB64 = message.delta;
|
|
90
|
+
const audioBytes = Buffer.from(audioB64, 'base64');
|
|
91
|
+
onAudioHandler?.(new Uint8Array(audioBytes));
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
case 'conversation.item.input_audio_transcription.completed': {
|
|
95
|
+
// User speech transcription
|
|
96
|
+
const transcript = message.transcript;
|
|
97
|
+
onTranscriptionHandler?.({
|
|
98
|
+
role: 'user',
|
|
99
|
+
text: transcript,
|
|
100
|
+
final: true,
|
|
101
|
+
});
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
case 'response.audio_transcript.delta': {
|
|
105
|
+
// Assistant speech transcription (streaming)
|
|
106
|
+
const delta = message.delta;
|
|
107
|
+
onTranscriptionHandler?.({
|
|
108
|
+
role: 'assistant',
|
|
109
|
+
text: delta,
|
|
110
|
+
final: false,
|
|
111
|
+
});
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
case 'response.audio_transcript.done': {
|
|
115
|
+
// Assistant transcription complete
|
|
116
|
+
const transcript = message.transcript;
|
|
117
|
+
onTranscriptionHandler?.({
|
|
118
|
+
role: 'assistant',
|
|
119
|
+
text: transcript,
|
|
120
|
+
final: true,
|
|
121
|
+
});
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
case 'response.done': {
|
|
125
|
+
onTurnCompleteHandler?.();
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
case 'input_audio_buffer.speech_started': {
|
|
129
|
+
// User started speaking, interrupt if needed
|
|
130
|
+
onInterruptedHandler?.();
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
case 'error': {
|
|
134
|
+
const errorMsg = message.error?.message || 'Unknown error';
|
|
135
|
+
onErrorHandler?.(new Error(String(errorMsg)));
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
ws.on('close', () => {
|
|
141
|
+
onErrorHandler?.(new Error('WebSocket connection closed'));
|
|
142
|
+
});
|
|
143
|
+
return {
|
|
144
|
+
send(message) {
|
|
145
|
+
if (ws.readyState !== WebSocket.OPEN)
|
|
146
|
+
return;
|
|
147
|
+
if (message.type === 'audio') {
|
|
148
|
+
// Send audio as base64 encoded PCM
|
|
149
|
+
const audioB64 = Buffer.from(message.data).toString('base64');
|
|
150
|
+
ws.send(JSON.stringify({
|
|
151
|
+
type: 'input_audio_buffer.append',
|
|
152
|
+
audio: audioB64,
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
else if (message.type === 'text') {
|
|
156
|
+
// Send text input
|
|
157
|
+
ws.send(JSON.stringify({
|
|
158
|
+
type: 'conversation.item.create',
|
|
159
|
+
item: {
|
|
160
|
+
type: 'message',
|
|
161
|
+
role: 'user',
|
|
162
|
+
content: [{ type: 'input_text', text: message.content }],
|
|
163
|
+
},
|
|
164
|
+
}));
|
|
165
|
+
ws.send(JSON.stringify({ type: 'response.create' }));
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
onAudio(handler) {
|
|
169
|
+
onAudioHandler = handler;
|
|
170
|
+
},
|
|
171
|
+
onTranscription(handler) {
|
|
172
|
+
onTranscriptionHandler = handler;
|
|
173
|
+
},
|
|
174
|
+
onTurnComplete(handler) {
|
|
175
|
+
onTurnCompleteHandler = handler;
|
|
176
|
+
},
|
|
177
|
+
onInterrupted(handler) {
|
|
178
|
+
onInterruptedHandler = handler;
|
|
179
|
+
},
|
|
180
|
+
onError(handler) {
|
|
181
|
+
onErrorHandler = handler;
|
|
182
|
+
},
|
|
183
|
+
interrupt() {
|
|
184
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
185
|
+
ws.send(JSON.stringify({ type: 'response.cancel' }));
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
async disconnect() {
|
|
189
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
190
|
+
ws.close();
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
//# sourceMappingURL=openai.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,kDAAkD;AAElD,OAAO,SAAS,MAAM,IAAI,CAAC;AAe3B,uCAAuC;AACvC,MAAM,aAAa,GAAkB;IACjC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE;IAC9B,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC5B,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE;IAC9B,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC5B,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC5B,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;CACrC,CAAC;AAEF,+CAA+C;AAC/C,MAAM,UAAU,MAAM,CAAC,OAA8B;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,yBAAyB,CAAC;IAEzD,OAAO;QACH,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,aAAa;QAErB,KAAK,CAAC,OAAO,CAAC,MAA6B;YACvC,OAAO,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACjE,CAAC;KACJ,CAAC;AACN,CAAC;AAED,0DAA0D;AAC1D,KAAK,UAAU,sBAAsB,CACjC,MAAc,EACd,KAAa,EACb,MAA6B;IAE7B,MAAM,GAAG,GAAG,0CAA0C,KAAK,EAAE,CAAC;IAE9D,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;QAC1B,OAAO,EAAE;YACL,eAAe,EAAE,UAAU,MAAM,EAAE;YACnC,aAAa,EAAE,aAAa;SAC/B;KACJ,CAAC,CAAC;IAEH,mCAAmC;IACnC,IAAI,cAAc,GAAyC,IAAI,CAAC;IAChE,IAAI,sBAAsB,GAAwC,IAAI,CAAC;IACvE,IAAI,qBAAqB,GAAwB,IAAI,CAAC;IACtD,IAAI,oBAAoB,GAAwB,IAAI,CAAC;IACrD,IAAI,cAAc,GAAoC,IAAI,CAAC;IAE3D,8BAA8B;IAC9B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACf,6BAA6B;YAC7B,MAAM,aAAa,GAA4B;gBAC3C,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;gBAC7B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,OAAO;gBAC9B,kBAAkB,EAAE,OAAO;gBAC3B,mBAAmB,EAAE,OAAO;gBAC5B,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;aACzC,CAAC;YAEF,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,aAAa,CAAC,YAAY,GAAG,MAAM,CAAC,iBAAiB,CAAC;YAC1D,CAAC;YAED,IAAI,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC;gBAC9B,aAAa,CAAC,yBAAyB,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;YACrE,CAAC;YAED,6BAA6B;YAC7B,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC9C,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC9B,CAAC,CAAC,CAAC;YACR,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;gBACnB,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,aAAa;aACzB,CAAC,CAAC,CAAC;YAEJ,OAAO,EAAE,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,uCAAuC;IACvC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;QAC9B,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC5C,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,cAAc,EAAE,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,mBAAmB,CAAC,OAAgC;QACzD,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,sBAAsB,CAAC,CAAC,CAAC;gBAC1B,uBAAuB;gBACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAe,CAAC;gBACzC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACnD,cAAc,EAAE,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC7C,MAAM;YACV,CAAC;YAED,KAAK,uDAAuD,CAAC,CAAC,CAAC;gBAC3D,4BAA4B;gBAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAoB,CAAC;gBAChD,sBAAsB,EAAE,CAAC;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,KAAK,EAAE,IAAI;iBACd,CAAC,CAAC;gBACH,MAAM;YACV,CAAC;YAED,KAAK,iCAAiC,CAAC,CAAC,CAAC;gBACrC,6CAA6C;gBAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAe,CAAC;gBACtC,sBAAsB,EAAE,CAAC;oBACrB,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,KAAK;iBACf,CAAC,CAAC;gBACH,MAAM;YACV,CAAC;YAED,KAAK,gCAAgC,CAAC,CAAC,CAAC;gBACpC,mCAAmC;gBACnC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAoB,CAAC;gBAChD,sBAAsB,EAAE,CAAC;oBACrB,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,UAAU;oBAChB,KAAK,EAAE,IAAI;iBACd,CAAC,CAAC;gBACH,MAAM;YACV,CAAC;YAED,KAAK,eAAe,CAAC,CAAC,CAAC;gBACnB,qBAAqB,EAAE,EAAE,CAAC;gBAC1B,MAAM;YACV,CAAC;YAED,KAAK,mCAAmC,CAAC,CAAC,CAAC;gBACvC,6CAA6C;gBAC7C,oBAAoB,EAAE,EAAE,CAAC;gBACzB,MAAM;YACV,CAAC;YAED,KAAK,OAAO,CAAC,CAAC,CAAC;gBACX,MAAM,QAAQ,GAAI,OAAO,CAAC,KAAiC,EAAE,OAAO,IAAI,eAAe,CAAC;gBACxF,cAAc,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC9C,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,cAAc,EAAE,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,OAAO;QACH,IAAI,CAAC,OAAwB;YACzB,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;gBAAE,OAAO;YAE7C,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC3B,mCAAmC;gBACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAC9D,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI,EAAE,2BAA2B;oBACjC,KAAK,EAAE,QAAQ;iBAClB,CAAC,CAAC,CAAC;YACR,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACjC,kBAAkB;gBAClB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI,EAAE,0BAA0B;oBAChC,IAAI,EAAE;wBACF,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;qBAC3D;iBACJ,CAAC,CAAC,CAAC;gBACJ,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO;YACX,cAAc,GAAG,OAAO,CAAC;QAC7B,CAAC;QAED,eAAe,CAAC,OAAO;YACnB,sBAAsB,GAAG,OAAO,CAAC;QACrC,CAAC;QAED,cAAc,CAAC,OAAO;YAClB,qBAAqB,GAAG,OAAO,CAAC;QACpC,CAAC;QAED,aAAa,CAAC,OAAO;YACjB,oBAAoB,GAAG,OAAO,CAAC;QACnC,CAAC;QAED,OAAO,CAAC,OAAO;YACX,cAAc,GAAG,OAAO,CAAC;QAC7B,CAAC;QAED,SAAS;YACL,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAED,KAAK,CAAC,UAAU;YACZ,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,EAAE,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACL,CAAC;KACJ,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../../src/server/auth.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAE5C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAqC9C,wBAAsB,YAAY,CAAC,KAAK,EACpC,GAAG,EAAE,eAAe,EACpB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,GAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAuBvB"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Authentication utilities
|
|
2
|
+
// Handles JWT validation and custom auth callbacks
|
|
3
|
+
import jwt from 'jsonwebtoken';
|
|
4
|
+
// Extracts auth token from request (Authorization header or query param)
|
|
5
|
+
function extractToken(req) {
|
|
6
|
+
// Check Authorization header first
|
|
7
|
+
const authHeader = req.headers['authorization'];
|
|
8
|
+
if (authHeader?.startsWith('Bearer ')) {
|
|
9
|
+
return authHeader.slice(7);
|
|
10
|
+
}
|
|
11
|
+
// Fallback to query parameter
|
|
12
|
+
const url = new URL(req.url ?? '', `http://${req.headers.host}`);
|
|
13
|
+
return url.searchParams.get('token');
|
|
14
|
+
}
|
|
15
|
+
// Verifies a JWT token using the provided secret
|
|
16
|
+
function verifyJwt(token, secret, algorithms) {
|
|
17
|
+
try {
|
|
18
|
+
const decoded = jwt.verify(token, secret, {
|
|
19
|
+
algorithms: algorithms ?? ['HS256'],
|
|
20
|
+
});
|
|
21
|
+
if (typeof decoded === 'string') {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
return decoded;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
// Authenticates a WebSocket upgrade request
|
|
31
|
+
export async function authenticate(req, authConfig) {
|
|
32
|
+
const token = extractToken(req);
|
|
33
|
+
if (!token)
|
|
34
|
+
return null;
|
|
35
|
+
let decoded = {};
|
|
36
|
+
if (authConfig.jwt) {
|
|
37
|
+
// Verify and decode JWT
|
|
38
|
+
const payload = verifyJwt(token, authConfig.jwt.secret, authConfig.jwt.algorithms);
|
|
39
|
+
if (!payload)
|
|
40
|
+
return null;
|
|
41
|
+
decoded = payload;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
// No JWT config, pass the raw token for custom handling
|
|
45
|
+
decoded = { token };
|
|
46
|
+
}
|
|
47
|
+
// Run custom validation callback
|
|
48
|
+
return authConfig.validate(decoded, req);
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=auth.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../src/server/auth.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,mDAAmD;AAGnD,OAAO,GAAG,MAAM,cAAc,CAAC;AAG/B,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAAoB;IACtC,mCAAmC;IACnC,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAChD,IAAI,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,8BAA8B;IAC9B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,OAAO,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,iDAAiD;AACjD,SAAS,SAAS,CACd,KAAa,EACb,MAAc,EACd,UAA4B;IAE5B,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE;YACtC,UAAU,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC;SACtC,CAAC,CAAC;QAEH,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED,4CAA4C;AAC5C,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,GAAoB,EACpB,UAA6B;IAE7B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,IAAI,OAAO,GAA4B,EAAE,CAAC;IAE1C,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC;QACjB,wBAAwB;QACxB,MAAM,OAAO,GAAG,SAAS,CACrB,KAAK,EACL,UAAU,CAAC,GAAG,CAAC,MAAM,EACrB,UAAU,CAAC,GAAG,CAAC,UAAyC,CAC3D,CAAC;QAEF,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,OAAO,GAAG,OAAkC,CAAC;IACjD,CAAC;SAAM,CAAC;QACJ,wDAAwD;QACxD,OAAO,GAAG,EAAE,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,iCAAiC;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type Server as HttpServer } from 'http';
|
|
2
|
+
import type { LukeServerConfig } from '../types.js';
|
|
3
|
+
export declare function createLukeServer<TUser, TSession>(config: LukeServerConfig<TUser, TSession>): LukeServerInstance;
|
|
4
|
+
export interface LukeServerInstance {
|
|
5
|
+
listen(port: number, callback?: () => void): void;
|
|
6
|
+
close(): Promise<void>;
|
|
7
|
+
readonly httpServer: HttpServer;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=ws-server.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ws-server.d.ts","sourceRoot":"","sources":["../../src/server/ws-server.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,KAAK,MAAM,IAAI,UAAU,EAAwB,MAAM,MAAM,CAAC;AAGrF,OAAO,KAAK,EACR,gBAAgB,EAQnB,MAAM,aAAa,CAAC;AAQrB,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,QAAQ,EAC5C,MAAM,EAAE,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC,GAC1C,kBAAkB,CAuTpB;AAED,MAAM,WAAW,kBAAkB;IAC/B,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAClD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CACnC"}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
// WebSocket Server
|
|
2
|
+
// Main server that handles client connections, authentication, and provider routing
|
|
3
|
+
import { createServer } from 'http';
|
|
4
|
+
import { WebSocketServer, WebSocket } from 'ws';
|
|
5
|
+
import { authenticate } from './auth.js';
|
|
6
|
+
// Generates a unique session ID
|
|
7
|
+
function generateSessionId() {
|
|
8
|
+
return `luke_${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
|
|
9
|
+
}
|
|
10
|
+
// Creates the Luke WebSocket server
|
|
11
|
+
export function createLukeServer(config) {
|
|
12
|
+
const httpServer = createServer();
|
|
13
|
+
const wss = new WebSocketServer({ noServer: true });
|
|
14
|
+
// Track active sessions by connection
|
|
15
|
+
const sessions = new Map();
|
|
16
|
+
const users = new Map();
|
|
17
|
+
// Handle WebSocket upgrade with auth
|
|
18
|
+
httpServer.on('upgrade', async (req, socket, head) => {
|
|
19
|
+
try {
|
|
20
|
+
const user = await authenticate(req, config.auth);
|
|
21
|
+
if (!user) {
|
|
22
|
+
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
|
23
|
+
socket.destroy();
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
wss.handleUpgrade(req, socket, head, (ws) => {
|
|
27
|
+
users.set(ws, user);
|
|
28
|
+
wss.emit('connection', ws, req, user);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
socket.write('HTTP/1.1 500 Internal Server Error\r\n\r\n');
|
|
33
|
+
socket.destroy();
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
wss.on('connection', async (ws, req, user) => {
|
|
37
|
+
// Send handshake with available providers
|
|
38
|
+
const handshake = {
|
|
39
|
+
type: 'handshake',
|
|
40
|
+
providers: config.providers.map((p) => ({
|
|
41
|
+
id: p.id,
|
|
42
|
+
name: p.name,
|
|
43
|
+
sampleRate: p.sampleRate,
|
|
44
|
+
voices: p.voices,
|
|
45
|
+
})),
|
|
46
|
+
defaultProvider: config.providers[0]?.id,
|
|
47
|
+
};
|
|
48
|
+
ws.send(JSON.stringify(handshake));
|
|
49
|
+
// Initialize session state
|
|
50
|
+
const session = {
|
|
51
|
+
id: generateSessionId(),
|
|
52
|
+
providerId: '',
|
|
53
|
+
providerConnection: null,
|
|
54
|
+
userSession: null,
|
|
55
|
+
createdAt: new Date(),
|
|
56
|
+
};
|
|
57
|
+
sessions.set(ws, session);
|
|
58
|
+
// Handle incoming messages
|
|
59
|
+
ws.on('message', async (data) => {
|
|
60
|
+
try {
|
|
61
|
+
const message = parseClientMessage(data);
|
|
62
|
+
if (!message)
|
|
63
|
+
return;
|
|
64
|
+
await handleClientMessage(ws, message, session, user);
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
sendError(ws, 'MESSAGE_ERROR', err instanceof Error ? err.message : 'Unknown error');
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
ws.on('close', async () => {
|
|
71
|
+
await cleanupSession(ws, session, user, 'disconnect');
|
|
72
|
+
});
|
|
73
|
+
ws.on('error', async (err) => {
|
|
74
|
+
await cleanupSession(ws, session, user, 'error');
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
// Parse raw message data into ClientMessage
|
|
78
|
+
function parseClientMessage(data) {
|
|
79
|
+
// Convert to string if Buffer
|
|
80
|
+
let stringData = null;
|
|
81
|
+
if (data instanceof Buffer) {
|
|
82
|
+
// Try to parse as JSON first (text messages come as Buffer too)
|
|
83
|
+
try {
|
|
84
|
+
stringData = data.toString('utf-8');
|
|
85
|
+
// Check if it looks like JSON
|
|
86
|
+
if (stringData.startsWith('{') || stringData.startsWith('[')) {
|
|
87
|
+
const parsed = JSON.parse(stringData);
|
|
88
|
+
return parsed;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
// Not JSON, treat as binary audio
|
|
93
|
+
}
|
|
94
|
+
// Binary audio data
|
|
95
|
+
return { type: 'audio', data: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) };
|
|
96
|
+
}
|
|
97
|
+
// String data
|
|
98
|
+
if (typeof data === 'string') {
|
|
99
|
+
try {
|
|
100
|
+
return JSON.parse(data);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
// Handle different client message types
|
|
109
|
+
async function handleClientMessage(ws, message, session, user) {
|
|
110
|
+
switch (message.type) {
|
|
111
|
+
case 'select_provider':
|
|
112
|
+
await handleSelectProvider(ws, message.providerId, message.voiceId, session, user);
|
|
113
|
+
break;
|
|
114
|
+
case 'audio':
|
|
115
|
+
if (session.providerConnection) {
|
|
116
|
+
session.providerConnection.send({
|
|
117
|
+
type: 'audio',
|
|
118
|
+
data: new Uint8Array(message.data),
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
break;
|
|
122
|
+
case 'text':
|
|
123
|
+
if (session.providerConnection) {
|
|
124
|
+
session.providerConnection.send({
|
|
125
|
+
type: 'text',
|
|
126
|
+
content: message.content,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
break;
|
|
130
|
+
case 'interrupt':
|
|
131
|
+
session.providerConnection?.interrupt();
|
|
132
|
+
break;
|
|
133
|
+
case 'reconnect':
|
|
134
|
+
// Attempt to resume previous session
|
|
135
|
+
if (config.session?.resolve) {
|
|
136
|
+
const existing = await config.session.resolve({ url: `/?sessionId=${message.sessionId}` }, user);
|
|
137
|
+
if (existing) {
|
|
138
|
+
session.userSession = existing;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// Connect to selected provider
|
|
145
|
+
async function handleSelectProvider(ws, providerId, voiceId, session, user) {
|
|
146
|
+
// Find the requested provider
|
|
147
|
+
const provider = config.providers.find((p) => p.id === providerId);
|
|
148
|
+
if (!provider) {
|
|
149
|
+
sendError(ws, 'INVALID_PROVIDER', `Provider ${providerId} not found`);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
// Disconnect existing provider connection if any
|
|
153
|
+
if (session.providerConnection) {
|
|
154
|
+
// Store reference and clear it before disconnecting
|
|
155
|
+
// This prevents the error handler from firing and confusing the client
|
|
156
|
+
const oldConnection = session.providerConnection;
|
|
157
|
+
session.providerConnection = null;
|
|
158
|
+
await oldConnection.disconnect();
|
|
159
|
+
}
|
|
160
|
+
// Create or resolve user session
|
|
161
|
+
if (config.session?.create && !session.userSession) {
|
|
162
|
+
session.userSession = await config.session.create(user, provider);
|
|
163
|
+
}
|
|
164
|
+
// Connect to the provider
|
|
165
|
+
try {
|
|
166
|
+
const connection = await provider.connect({
|
|
167
|
+
voice: voiceId ?? provider.voices[0]?.id,
|
|
168
|
+
systemInstruction: config.config?.systemInstruction,
|
|
169
|
+
transcription: config.config?.transcription,
|
|
170
|
+
tools: config.config?.tools,
|
|
171
|
+
});
|
|
172
|
+
session.providerId = providerId;
|
|
173
|
+
session.providerConnection = connection;
|
|
174
|
+
// Wire up provider events to client
|
|
175
|
+
setupProviderHandlers(ws, connection, session);
|
|
176
|
+
// Notify client that session is ready
|
|
177
|
+
const readyMsg = {
|
|
178
|
+
type: 'session_ready',
|
|
179
|
+
sessionId: session.id,
|
|
180
|
+
sampleRate: provider.sampleRate,
|
|
181
|
+
};
|
|
182
|
+
ws.send(JSON.stringify(readyMsg));
|
|
183
|
+
// Trigger onConnect callback
|
|
184
|
+
config.onConnect?.(session, user);
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
sendError(ws, 'PROVIDER_ERROR', err instanceof Error ? err.message : 'Connection failed');
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
// Set up event handlers from provider to client
|
|
191
|
+
function setupProviderHandlers(ws, connection, session) {
|
|
192
|
+
connection.onAudio((audio) => {
|
|
193
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
194
|
+
// Send audio as binary frame
|
|
195
|
+
ws.send(audio);
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
connection.onTranscription((transcription) => {
|
|
199
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
200
|
+
const msg = {
|
|
201
|
+
type: 'transcription',
|
|
202
|
+
role: transcription.role,
|
|
203
|
+
text: transcription.text,
|
|
204
|
+
final: transcription.final,
|
|
205
|
+
};
|
|
206
|
+
ws.send(JSON.stringify(msg));
|
|
207
|
+
}
|
|
208
|
+
// Call transcription callback
|
|
209
|
+
config.onTranscription?.(transcription, session);
|
|
210
|
+
});
|
|
211
|
+
connection.onTurnComplete(() => {
|
|
212
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
213
|
+
ws.send(JSON.stringify({ type: 'turn_complete' }));
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
connection.onInterrupted(() => {
|
|
217
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
218
|
+
ws.send(JSON.stringify({ type: 'interrupted' }));
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
connection.onError((error) => {
|
|
222
|
+
// Only send error if this connection is still the active one
|
|
223
|
+
// (prevents errors when switching providers)
|
|
224
|
+
if (session.providerConnection === connection) {
|
|
225
|
+
sendError(ws, 'PROVIDER_ERROR', error.message);
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
// Clean up when client disconnects
|
|
230
|
+
async function cleanupSession(ws, session, user, reason) {
|
|
231
|
+
if (session.providerConnection) {
|
|
232
|
+
await session.providerConnection.disconnect();
|
|
233
|
+
}
|
|
234
|
+
if (config.session?.onEnd && session.userSession) {
|
|
235
|
+
await config.session.onEnd(session.userSession, reason);
|
|
236
|
+
}
|
|
237
|
+
config.onDisconnect?.(session, user);
|
|
238
|
+
sessions.delete(ws);
|
|
239
|
+
users.delete(ws);
|
|
240
|
+
}
|
|
241
|
+
// Send error message to client
|
|
242
|
+
function sendError(ws, code, message) {
|
|
243
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
244
|
+
const msg = { type: 'error', code, message };
|
|
245
|
+
ws.send(JSON.stringify(msg));
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return {
|
|
249
|
+
listen(port, callback) {
|
|
250
|
+
httpServer.listen(port, callback);
|
|
251
|
+
},
|
|
252
|
+
close() {
|
|
253
|
+
return new Promise((resolve) => {
|
|
254
|
+
wss.close(() => {
|
|
255
|
+
httpServer.close(() => resolve());
|
|
256
|
+
});
|
|
257
|
+
});
|
|
258
|
+
},
|
|
259
|
+
get httpServer() {
|
|
260
|
+
return httpServer;
|
|
261
|
+
},
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
//# sourceMappingURL=ws-server.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ws-server.js","sourceRoot":"","sources":["../../src/server/ws-server.ts"],"names":[],"mappings":"AAAA,mBAAmB;AACnB,oFAAoF;AAEpF,OAAO,EAAE,YAAY,EAAmD,MAAM,MAAM,CAAC;AACrF,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAYzC,gCAAgC;AAChC,SAAS,iBAAiB;IACtB,OAAO,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAC3E,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,gBAAgB,CAC5B,MAAyC;IAEzC,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,eAAe,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpD,sCAAsC;IACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC7D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE1C,qCAAqC;IACrC,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACjD,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAClD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAClD,MAAM,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;gBACxC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBACpB,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAC1C,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,EAAE,EAAa,EAAE,GAAoB,EAAE,IAAW,EAAE,EAAE;QAE5E,0CAA0C;QAC1C,MAAM,SAAS,GAAqB;YAChC,IAAI,EAAE,WAAW;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACnB,CAAC,CAAC;YACH,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE;SAC3C,CAAC;QACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAEnC,2BAA2B;QAC3B,MAAM,OAAO,GAA0B;YACnC,EAAE,EAAE,iBAAiB,EAAE;YACvB,UAAU,EAAE,EAAE;YACd,kBAAkB,EAAE,IAAI;YACxB,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI,IAAI,EAAE;SACxB,CAAC;QACF,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAE1B,2BAA2B;QAC3B,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,OAAO;oBAAE,OAAO;gBAErB,MAAM,mBAAmB,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAC1D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,SAAS,CAAC,EAAE,EAAE,eAAe,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;YACzF,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YACtB,MAAM,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YACzB,MAAM,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,4CAA4C;IAC5C,SAAS,kBAAkB,CAAC,IAAa;QACrC,8BAA8B;QAC9B,IAAI,UAAU,GAAkB,IAAI,CAAC;QAErC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,gEAAgE;YAChE,IAAI,CAAC;gBACD,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpC,8BAA8B;gBAC9B,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAkB,CAAC;oBACvD,OAAO,MAAM,CAAC;gBAClB,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACL,kCAAkC;YACtC,CAAC;YAED,oBAAoB;YACpB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1G,CAAC;QAED,cAAc;QACd,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,wCAAwC;IACxC,KAAK,UAAU,mBAAmB,CAC9B,EAAa,EACb,OAAsB,EACtB,OAA8B,EAC9B,IAAW;QAEX,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,iBAAiB;gBAClB,MAAM,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;gBACnF,MAAM;YAEV,KAAK,OAAO;gBACR,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;oBAC7B,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC;wBAC5B,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;qBACrC,CAAC,CAAC;gBACP,CAAC;gBACD,MAAM;YAEV,KAAK,MAAM;gBACP,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;oBAC7B,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC;wBAC5B,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,OAAO,CAAC,OAAO;qBAC3B,CAAC,CAAC;gBACP,CAAC;gBACD,MAAM;YAEV,KAAK,WAAW;gBACZ,OAAO,CAAC,kBAAkB,EAAE,SAAS,EAAE,CAAC;gBACxC,MAAM;YAEV,KAAK,WAAW;gBACZ,qCAAqC;gBACrC,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;oBAC1B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CACzC,EAAE,GAAG,EAAE,eAAe,OAAO,CAAC,SAAS,EAAE,EAAqB,EAC9D,IAAI,CACP,CAAC;oBACF,IAAI,QAAQ,EAAE,CAAC;wBACX,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC;oBACnC,CAAC;gBACL,CAAC;gBACD,MAAM;QACd,CAAC;IACL,CAAC;IAED,+BAA+B;IAC/B,KAAK,UAAU,oBAAoB,CAC/B,EAAa,EACb,UAAkB,EAClB,OAA2B,EAC3B,OAA8B,EAC9B,IAAW;QAEX,8BAA8B;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,SAAS,CAAC,EAAE,EAAE,kBAAkB,EAAE,YAAY,UAAU,YAAY,CAAC,CAAC;YACtE,OAAO;QACX,CAAC;QAED,iDAAiD;QACjD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAC7B,oDAAoD;YACpD,uEAAuE;YACvE,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;YACjD,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAClC,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;QACrC,CAAC;QAED,iCAAiC;QACjC,IAAI,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,CAAC,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtE,CAAC;QAED,0BAA0B;QAC1B,IAAI,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC;gBACtC,KAAK,EAAE,OAAO,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB;gBACnD,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa;gBAC3C,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK;aAC9B,CAAC,CAAC;YAEH,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;YAChC,OAAO,CAAC,kBAAkB,GAAG,UAAU,CAAC;YAExC,oCAAoC;YACpC,qBAAqB,CAAC,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YAE/C,sCAAsC;YACtC,MAAM,QAAQ,GAAkB;gBAC5B,IAAI,EAAE,eAAe;gBACrB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;aAClC,CAAC;YACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;YAElC,6BAA6B;YAC7B,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,SAAS,CAAC,EAAE,EAAE,gBAAgB,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC9F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,SAAS,qBAAqB,CAC1B,EAAa,EACb,UAA8B,EAC9B,OAA8B;QAE9B,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACzB,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,6BAA6B;gBAC7B,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,eAAe,CAAC,CAAC,aAA4B,EAAE,EAAE;YACxD,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAkB;oBACvB,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,KAAK,EAAE,aAAa,CAAC,KAAK;iBAC7B,CAAC;gBACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,CAAC;YAED,8BAA8B;YAC9B,MAAM,CAAC,eAAe,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,cAAc,CAAC,GAAG,EAAE;YAC3B,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,aAAa,CAAC,GAAG,EAAE;YAC1B,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACzB,6DAA6D;YAC7D,6CAA6C;YAC7C,IAAI,OAAO,CAAC,kBAAkB,KAAK,UAAU,EAAE,CAAC;gBAC5C,SAAS,CAAC,EAAE,EAAE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,mCAAmC;IACnC,KAAK,UAAU,cAAc,CACzB,EAAa,EACb,OAA8B,EAC9B,IAAW,EACX,MAA0C;QAE1C,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAC7B,MAAM,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC;QAClD,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAErC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrB,CAAC;IAED,+BAA+B;IAC/B,SAAS,SAAS,CAAC,EAAa,EAAE,IAAY,EAAE,OAAe;QAC3D,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,GAAG,GAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YAC5D,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;IAED,OAAO;QACH,MAAM,CAAC,IAAY,EAAE,QAAqB;YACtC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,KAAK;YACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC3B,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE;oBACX,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,UAAU;YACV,OAAO,UAAU,CAAC;QACtB,CAAC;KACJ,CAAC;AACN,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import type { IncomingMessage } from 'http';
|
|
2
|
+
import type { z } from 'zod';
|
|
3
|
+
export type ProviderName = 'openai' | 'gemini';
|
|
4
|
+
export interface VoiceConfig {
|
|
5
|
+
id: string;
|
|
6
|
+
name: string;
|
|
7
|
+
language?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface LukeProvider {
|
|
10
|
+
readonly id: string;
|
|
11
|
+
readonly name: ProviderName;
|
|
12
|
+
readonly sampleRate: 24000 | 16000;
|
|
13
|
+
readonly voices: VoiceConfig[];
|
|
14
|
+
connect(config: ProviderSessionConfig): Promise<ProviderConnection>;
|
|
15
|
+
}
|
|
16
|
+
export interface ProviderSessionConfig {
|
|
17
|
+
model?: string;
|
|
18
|
+
voice?: string;
|
|
19
|
+
systemInstruction?: string;
|
|
20
|
+
tools?: ToolDefinition[];
|
|
21
|
+
transcription?: {
|
|
22
|
+
input?: boolean;
|
|
23
|
+
output?: boolean;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export interface ProviderConnection {
|
|
27
|
+
send(message: ProviderMessage): void;
|
|
28
|
+
onAudio(handler: (audio: Uint8Array) => void): void;
|
|
29
|
+
onTranscription(handler: (transcription: Transcription) => void): void;
|
|
30
|
+
onTurnComplete(handler: () => void): void;
|
|
31
|
+
onInterrupted(handler: () => void): void;
|
|
32
|
+
onError(handler: (error: Error) => void): void;
|
|
33
|
+
interrupt(): void;
|
|
34
|
+
disconnect(): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
export type ProviderMessage = {
|
|
37
|
+
type: 'audio';
|
|
38
|
+
data: Uint8Array;
|
|
39
|
+
} | {
|
|
40
|
+
type: 'text';
|
|
41
|
+
content: string;
|
|
42
|
+
};
|
|
43
|
+
export interface Transcription {
|
|
44
|
+
role: 'user' | 'assistant';
|
|
45
|
+
text: string;
|
|
46
|
+
final: boolean;
|
|
47
|
+
}
|
|
48
|
+
export interface ToolDefinition<T = unknown> {
|
|
49
|
+
name: string;
|
|
50
|
+
description: string;
|
|
51
|
+
parameters: z.ZodType<T>;
|
|
52
|
+
execute: (params: T) => Promise<unknown>;
|
|
53
|
+
}
|
|
54
|
+
export interface JwtConfig {
|
|
55
|
+
secret: string;
|
|
56
|
+
algorithms?: string[];
|
|
57
|
+
}
|
|
58
|
+
export interface AuthConfig<TUser = unknown> {
|
|
59
|
+
jwt?: JwtConfig;
|
|
60
|
+
validate: (decoded: Record<string, unknown>, req: IncomingMessage) => Promise<TUser | null>;
|
|
61
|
+
}
|
|
62
|
+
export interface SessionConfig<TSession = unknown, TUser = unknown> {
|
|
63
|
+
resolve?: (req: IncomingMessage, user: TUser) => Promise<TSession | null>;
|
|
64
|
+
create?: (user: TUser, provider: LukeProvider) => Promise<TSession>;
|
|
65
|
+
onEnd?: (session: TSession, reason: 'disconnect' | 'error' | 'timeout') => Promise<void>;
|
|
66
|
+
}
|
|
67
|
+
export interface LukeServerConfig<TUser = unknown, TSession = unknown> {
|
|
68
|
+
providers: LukeProvider[];
|
|
69
|
+
auth: AuthConfig<TUser>;
|
|
70
|
+
session?: SessionConfig<TSession, TUser>;
|
|
71
|
+
config?: Partial<ProviderSessionConfig>;
|
|
72
|
+
onConnect?: (session: LukeSession<TSession>, user: TUser) => void;
|
|
73
|
+
onDisconnect?: (session: LukeSession<TSession>, user: TUser) => void;
|
|
74
|
+
onTranscription?: (transcription: Transcription, session: LukeSession<TSession>) => void;
|
|
75
|
+
}
|
|
76
|
+
export interface LukeSession<TSession = unknown> {
|
|
77
|
+
id: string;
|
|
78
|
+
providerId: string;
|
|
79
|
+
providerConnection: ProviderConnection | null;
|
|
80
|
+
userSession: TSession | null;
|
|
81
|
+
createdAt: Date;
|
|
82
|
+
}
|
|
83
|
+
export interface HandshakeMessage {
|
|
84
|
+
type: 'handshake';
|
|
85
|
+
providers: Array<{
|
|
86
|
+
id: string;
|
|
87
|
+
name: ProviderName;
|
|
88
|
+
sampleRate: 16000 | 24000;
|
|
89
|
+
voices: VoiceConfig[];
|
|
90
|
+
}>;
|
|
91
|
+
defaultProvider?: string;
|
|
92
|
+
}
|
|
93
|
+
export type ClientMessage = {
|
|
94
|
+
type: 'select_provider';
|
|
95
|
+
providerId: string;
|
|
96
|
+
voiceId?: string;
|
|
97
|
+
} | {
|
|
98
|
+
type: 'audio';
|
|
99
|
+
data: ArrayBuffer;
|
|
100
|
+
} | {
|
|
101
|
+
type: 'text';
|
|
102
|
+
content: string;
|
|
103
|
+
} | {
|
|
104
|
+
type: 'interrupt';
|
|
105
|
+
} | {
|
|
106
|
+
type: 'reconnect';
|
|
107
|
+
sessionId: string;
|
|
108
|
+
};
|
|
109
|
+
export type ServerMessage = HandshakeMessage | {
|
|
110
|
+
type: 'session_ready';
|
|
111
|
+
sessionId: string;
|
|
112
|
+
sampleRate: number;
|
|
113
|
+
} | {
|
|
114
|
+
type: 'audio';
|
|
115
|
+
data: ArrayBuffer;
|
|
116
|
+
} | {
|
|
117
|
+
type: 'transcription';
|
|
118
|
+
role: 'user' | 'assistant';
|
|
119
|
+
text: string;
|
|
120
|
+
final: boolean;
|
|
121
|
+
} | {
|
|
122
|
+
type: 'turn_complete';
|
|
123
|
+
} | {
|
|
124
|
+
type: 'interrupted';
|
|
125
|
+
} | {
|
|
126
|
+
type: 'error';
|
|
127
|
+
code: string;
|
|
128
|
+
message: string;
|
|
129
|
+
};
|
|
130
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAC5C,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAG7B,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAG/C,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAGD,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,KAAK,GAAG,KAAK,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAC/B,OAAO,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvE;AAGD,MAAM,WAAW,qBAAqB;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,aAAa,CAAC,EAAE;QACZ,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;CACL;AAGD,MAAM,WAAW,kBAAkB;IAC/B,IAAI,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAAC;IACrC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IACpD,eAAe,CAAC,OAAO,EAAE,CAAC,aAAa,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IACvE,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAC1C,aAAa,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IACzC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IAC/C,SAAS,IAAI,IAAI,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B;AAGD,MAAM,MAAM,eAAe,GACrB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAGxC,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;CAClB;AAGD,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,OAAO;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5C;AAGD,MAAM,WAAW,SAAS;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAGD,MAAM,WAAW,UAAU,CAAC,KAAK,GAAG,OAAO;IACvC,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;CAC/F;AAGD,MAAM,WAAW,aAAa,CAAC,QAAQ,GAAG,OAAO,EAAE,KAAK,GAAG,OAAO;IAC9D,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;IAC1E,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,GAAG,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5F;AAGD,MAAM,WAAW,gBAAgB,CAAC,KAAK,GAAG,OAAO,EAAE,QAAQ,GAAG,OAAO;IACjE,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACxB,OAAO,CAAC,EAAE,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACxC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC;IAClE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC;IACrE,eAAe,CAAC,EAAE,CAAC,aAAa,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;CAC5F;AAGD,MAAM,WAAW,WAAW,CAAC,QAAQ,GAAG,OAAO;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC9C,WAAW,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,IAAI,CAAC;CACnB;AAGD,MAAM,WAAW,gBAAgB;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;QACb,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,YAAY,CAAC;QACnB,UAAU,EAAE,KAAK,GAAG,KAAK,CAAC;QAC1B,MAAM,EAAE,WAAW,EAAE,CAAC;KACzB,CAAC,CAAC;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAGD,MAAM,MAAM,aAAa,GACnB;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAG/C,MAAM,MAAM,aAAa,GACnB,gBAAgB,GAChB;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GACnF;IAAE,IAAI,EAAE,eAAe,CAAA;CAAE,GACzB;IAAE,IAAI,EAAE,aAAa,CAAA;CAAE,GACvB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,yEAAyE"}
|
package/package.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@leia-org/luke-server",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"publishConfig": {
|
|
5
|
+
"access": "public"
|
|
6
|
+
},
|
|
7
|
+
"description": "Unified realtime AI server for OpenAI and Gemini",
|
|
8
|
+
"type": "module",
|
|
9
|
+
"main": "./dist/index.js",
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"exports": {
|
|
12
|
+
".": {
|
|
13
|
+
"import": "./dist/index.js",
|
|
14
|
+
"types": "./dist/index.d.ts"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
"dependencies": {
|
|
18
|
+
"jsonwebtoken": "^9.0.2",
|
|
19
|
+
"ws": "^8.18.0",
|
|
20
|
+
"zod": "^3.24.1"
|
|
21
|
+
},
|
|
22
|
+
"devDependencies": {
|
|
23
|
+
"@types/jsonwebtoken": "^9.0.7",
|
|
24
|
+
"@types/node": "^22.10.2",
|
|
25
|
+
"@types/ws": "^8.5.13",
|
|
26
|
+
"typescript": "^5.7.2",
|
|
27
|
+
"vitest": "^2.1.8"
|
|
28
|
+
},
|
|
29
|
+
"peerDependencies": {},
|
|
30
|
+
"files": [
|
|
31
|
+
"dist"
|
|
32
|
+
],
|
|
33
|
+
"license": "MIT",
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "tsc",
|
|
36
|
+
"dev": "tsc --watch",
|
|
37
|
+
"test": "vitest run",
|
|
38
|
+
"lint": "tsc --noEmit"
|
|
39
|
+
}
|
|
40
|
+
}
|