@just-every/ensemble 0.2.74 → 0.2.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/ensemble_listen.d.ts +2 -6
- package/dist/core/ensemble_listen.d.ts.map +1 -1
- package/dist/core/ensemble_listen.js +110 -81
- package/dist/core/ensemble_listen.js.map +1 -1
- package/dist/data/model_data.d.ts +2 -2
- package/dist/data/model_data.d.ts.map +1 -1
- package/dist/data/model_data.js +14 -65
- package/dist/data/model_data.js.map +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/model_providers/gemini.d.ts +4 -1
- package/dist/model_providers/gemini.d.ts.map +1 -1
- package/dist/model_providers/gemini.js +220 -0
- package/dist/model_providers/gemini.js.map +1 -1
- package/dist/model_providers/model_provider.d.ts.map +1 -1
- package/dist/model_providers/model_provider.js +2 -8
- package/dist/model_providers/model_provider.js.map +1 -1
- package/dist/model_providers/openai.d.ts +1 -2
- package/dist/model_providers/openai.d.ts.map +1 -1
- package/dist/model_providers/openai.js +0 -150
- package/dist/model_providers/openai.js.map +1 -1
- package/dist/model_providers/test_provider.d.ts +0 -1
- package/dist/model_providers/test_provider.d.ts.map +1 -1
- package/dist/model_providers/test_provider.js +0 -19
- package/dist/model_providers/test_provider.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types/types.d.ts +60 -94
- package/dist/types/types.d.ts.map +1 -1
- package/dist/utils/cost_tracker.d.ts.map +1 -1
- package/dist/utils/cost_tracker.js +15 -3
- package/dist/utils/cost_tracker.js.map +1 -1
- package/package.json +18 -16
- package/dist/utils/audio_converter.d.ts +0 -11
- package/dist/utils/audio_converter.d.ts.map +0 -1
- package/dist/utils/audio_converter.js +0 -134
- package/dist/utils/audio_converter.js.map +0 -1
- package/dist/utils/gemini_live_client.d.ts +0 -8
- package/dist/utils/gemini_live_client.d.ts.map +0 -1
- package/dist/utils/gemini_live_client.js +0 -85
- package/dist/utils/gemini_live_client.js.map +0 -1
- package/dist/utils/openai_realtime_client.d.ts +0 -18
- package/dist/utils/openai_realtime_client.d.ts.map +0 -1
- package/dist/utils/openai_realtime_client.js +0 -238
- package/dist/utils/openai_realtime_client.js.map +0 -1
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
export async function* streamGeminiLive(audio, config) {
|
|
2
|
-
try {
|
|
3
|
-
const startEvent = {
|
|
4
|
-
type: 'transcription_start',
|
|
5
|
-
timestamp: new Date().toISOString(),
|
|
6
|
-
format: config.options?.response_format || 'text',
|
|
7
|
-
language: config.options?.language,
|
|
8
|
-
audio_format: {
|
|
9
|
-
sampleRate: 16000,
|
|
10
|
-
channels: 1,
|
|
11
|
-
bitDepth: 16,
|
|
12
|
-
encoding: 'pcm',
|
|
13
|
-
},
|
|
14
|
-
};
|
|
15
|
-
yield startEvent;
|
|
16
|
-
let audioData;
|
|
17
|
-
if (audio instanceof ReadableStream) {
|
|
18
|
-
const chunks = [];
|
|
19
|
-
const reader = audio.getReader();
|
|
20
|
-
try {
|
|
21
|
-
while (true) {
|
|
22
|
-
const { done, value } = await reader.read();
|
|
23
|
-
if (done)
|
|
24
|
-
break;
|
|
25
|
-
if (value)
|
|
26
|
-
chunks.push(value);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
finally {
|
|
30
|
-
reader.releaseLock();
|
|
31
|
-
}
|
|
32
|
-
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
|
|
33
|
-
audioData = new Uint8Array(totalLength);
|
|
34
|
-
let offset = 0;
|
|
35
|
-
for (const chunk of chunks) {
|
|
36
|
-
audioData.set(chunk, offset);
|
|
37
|
-
offset += chunk.length;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
else if (audio instanceof ArrayBuffer) {
|
|
41
|
-
audioData = new Uint8Array(audio);
|
|
42
|
-
}
|
|
43
|
-
else if (typeof audio === 'string') {
|
|
44
|
-
const binaryString = atob(audio);
|
|
45
|
-
audioData = new Uint8Array(binaryString.length);
|
|
46
|
-
for (let i = 0; i < binaryString.length; i++) {
|
|
47
|
-
audioData[i] = binaryString.charCodeAt(i);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
else {
|
|
51
|
-
throw new Error('Unsupported audio source type');
|
|
52
|
-
}
|
|
53
|
-
const placeholderText = '[Gemini Live transcription not yet implemented. Audio received: ' +
|
|
54
|
-
audioData.length +
|
|
55
|
-
' bytes]';
|
|
56
|
-
const completeEvent = {
|
|
57
|
-
type: 'transcription_complete',
|
|
58
|
-
timestamp: new Date().toISOString(),
|
|
59
|
-
text: placeholderText,
|
|
60
|
-
};
|
|
61
|
-
yield completeEvent;
|
|
62
|
-
}
|
|
63
|
-
catch (err) {
|
|
64
|
-
const errorEvent = {
|
|
65
|
-
type: 'error',
|
|
66
|
-
timestamp: new Date().toISOString(),
|
|
67
|
-
error: err instanceof Error ? err.message : 'Unknown error',
|
|
68
|
-
};
|
|
69
|
-
yield errorEvent;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
async function convertToPCM16ForGemini(audioData, audioFormat) {
|
|
73
|
-
const { AudioConverter } = await import('./audio_converter.js');
|
|
74
|
-
return AudioConverter.convertToPCM16(audioData, audioFormat, 16000);
|
|
75
|
-
}
|
|
76
|
-
async function sendRealtimeInput(session, audioData) {
|
|
77
|
-
console.log('[Gemini Live] Would send audio chunk:', audioData.length, 'bytes');
|
|
78
|
-
}
|
|
79
|
-
async function sendAudioStreamEnd(session) {
|
|
80
|
-
console.log('[Gemini Live] Would send stream end signal');
|
|
81
|
-
}
|
|
82
|
-
function extractTranscription(chunk) {
|
|
83
|
-
return null;
|
|
84
|
-
}
|
|
85
|
-
//# sourceMappingURL=gemini_live_client.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"gemini_live_client.js","sourceRoot":"","sources":["../../utils/gemini_live_client.ts"],"names":[],"mappings":"AAiCA,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CACnC,KAA+B,EAC/B,MAAwB;IAExB,IAAI,CAAC;QAED,MAAM,UAAU,GAA4B;YACxC,IAAI,EAAE,qBAAqB;YAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,IAAI,MAAM;YACjD,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ;YAClC,YAAY,EAAE;gBACV,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,CAAC;gBACX,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE,KAAK;aAClB;SACJ,CAAC;QACF,MAAM,UAAU,CAAC;QAGjB,IAAI,SAAqB,CAAC;QAE1B,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;YAClC,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YAEjC,IAAI,CAAC;gBACD,OAAO,IAAI,EAAE,CAAC;oBACV,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAChB,IAAI,KAAK;wBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;oBAAS,CAAC;gBACP,MAAM,CAAC,WAAW,EAAE,CAAC;YACzB,CAAC;YAGD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAC7B,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAClC,CAAC,CACJ,CAAC;YACF,SAAS,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;YACxC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,CAAC;QACL,CAAC;aAAM,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACtC,SAAS,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAEnC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,SAAS,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,SAAS,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACrD,CAAC;QAGD,MAAM,eAAe,GACjB,kEAAkE;YAClE,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC;QAGd,MAAM,aAAa,GAA+B;YAC9C,IAAI,EAAE,wBAAwB;YAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,eAAe;SACxB,CAAC;QACF,MAAM,aAAa,CAAC;IACxB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAC9D,CAAC;QACF,MAAM,UAAU,CAAC;IACrB,CAAC;AACL,CAAC;AAMD,KAAK,UAAU,uBAAuB,CAClC,SAAqB,EACrB,WAA+C;IAG/C,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAGhE,OAAO,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;AACxE,CAAC;AAOD,KAAK,UAAU,iBAAiB,CAE5B,OAAY,EACZ,SAAqB;IAGrB,OAAO,CAAC,GAAG,CACP,uCAAuC,EACvC,SAAS,CAAC,MAAM,EAChB,OAAO,CACV,CAAC;AACN,CAAC;AAOD,KAAK,UAAU,kBAAkB,CAE7B,OAAY;IAGZ,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;AAC9D,CAAC;AAOD,SAAS,oBAAoB,CAEzB,KAAU;IAGV,OAAO,IAAI,CAAC;AAChB,CAAC"}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import type { TranscriptionOpts, TranscriptionEvent, TranscriptionAudioSource } from '../types/types.js';
|
|
2
|
-
export interface OpenAIRealtimeConfig {
|
|
3
|
-
apiKey: string;
|
|
4
|
-
model?: string;
|
|
5
|
-
options?: TranscriptionOpts;
|
|
6
|
-
}
|
|
7
|
-
export interface OpenAIRealtimeMessage {
|
|
8
|
-
type: string;
|
|
9
|
-
event_id?: string;
|
|
10
|
-
session?: any;
|
|
11
|
-
conversation?: any;
|
|
12
|
-
audio?: string;
|
|
13
|
-
item?: any;
|
|
14
|
-
error?: any;
|
|
15
|
-
delta?: any;
|
|
16
|
-
}
|
|
17
|
-
export declare function streamOpenAIRealtime(audio: TranscriptionAudioSource, config: OpenAIRealtimeConfig): AsyncGenerator<TranscriptionEvent>;
|
|
18
|
-
//# sourceMappingURL=openai_realtime_client.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"openai_realtime_client.d.ts","sourceRoot":"","sources":["../../utils/openai_realtime_client.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EAM3B,MAAM,mBAAmB,CAAC;AAE3B,MAAM,WAAW,oBAAoB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,KAAK,CAAC,EAAE,GAAG,CAAC;CACf;AAKD,wBAAuB,oBAAoB,CACvC,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,oBAAoB,GAC7B,cAAc,CAAC,kBAAkB,CAAC,CAgSpC"}
|
|
@@ -1,238 +0,0 @@
|
|
|
1
|
-
let WebSocket;
|
|
2
|
-
if (typeof window !== 'undefined' && window.WebSocket) {
|
|
3
|
-
WebSocket = window.WebSocket;
|
|
4
|
-
}
|
|
5
|
-
else {
|
|
6
|
-
WebSocket = null;
|
|
7
|
-
}
|
|
8
|
-
export async function* streamOpenAIRealtime(audio, config) {
|
|
9
|
-
if (!WebSocket) {
|
|
10
|
-
try {
|
|
11
|
-
const wsModule = await import('ws');
|
|
12
|
-
WebSocket = wsModule.default;
|
|
13
|
-
}
|
|
14
|
-
catch {
|
|
15
|
-
throw new Error('WebSocket not available. In Node.js, install the "ws" package.');
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
const wsUrl = 'wss://api.openai.com/v1/realtime';
|
|
19
|
-
const model = config.model || 'gpt-4o-realtime-preview-2024-12-17';
|
|
20
|
-
const ws = new WebSocket(wsUrl, {
|
|
21
|
-
headers: {
|
|
22
|
-
Authorization: `Bearer ${config.apiKey}`,
|
|
23
|
-
'OpenAI-Beta': 'realtime=v1',
|
|
24
|
-
},
|
|
25
|
-
});
|
|
26
|
-
const eventQueue = [];
|
|
27
|
-
let wsConnected = false;
|
|
28
|
-
let sessionConfigured = false;
|
|
29
|
-
let audioStreamEnded = false;
|
|
30
|
-
let transcriptionComplete = false;
|
|
31
|
-
let currentTranscript = '';
|
|
32
|
-
let error = null;
|
|
33
|
-
ws.on('open', () => {
|
|
34
|
-
console.log('[OpenAI Realtime] WebSocket connected');
|
|
35
|
-
wsConnected = true;
|
|
36
|
-
const sessionConfig = {
|
|
37
|
-
type: 'session.update',
|
|
38
|
-
session: {
|
|
39
|
-
model: model,
|
|
40
|
-
modalities: ['text'],
|
|
41
|
-
instructions: 'Transcribe the audio input accurately.',
|
|
42
|
-
voice: 'echo',
|
|
43
|
-
input_audio_format: 'pcm16',
|
|
44
|
-
output_audio_format: 'pcm16',
|
|
45
|
-
input_audio_transcription: {
|
|
46
|
-
enabled: true,
|
|
47
|
-
model: 'whisper-1',
|
|
48
|
-
},
|
|
49
|
-
turn_detection: config.options?.vad?.enabled !== false
|
|
50
|
-
? {
|
|
51
|
-
type: config.options?.vad?.mode || 'server_vad',
|
|
52
|
-
threshold: config.options?.vad?.threshold || 0.5,
|
|
53
|
-
prefix_padding_ms: config.options?.vad?.prefix_padding_ms || 300,
|
|
54
|
-
silence_duration_ms: config.options?.vad?.silence_duration_ms ||
|
|
55
|
-
200,
|
|
56
|
-
}
|
|
57
|
-
: null,
|
|
58
|
-
temperature: config.options?.temperature || 0.8,
|
|
59
|
-
max_output_tokens: 'inf',
|
|
60
|
-
},
|
|
61
|
-
};
|
|
62
|
-
ws.send(JSON.stringify(sessionConfig));
|
|
63
|
-
sessionConfigured = true;
|
|
64
|
-
const startEvent = {
|
|
65
|
-
type: 'transcription_start',
|
|
66
|
-
timestamp: new Date().toISOString(),
|
|
67
|
-
format: config.options?.response_format || 'text',
|
|
68
|
-
language: config.options?.language,
|
|
69
|
-
audio_format: {
|
|
70
|
-
sampleRate: 24000,
|
|
71
|
-
channels: 1,
|
|
72
|
-
bitDepth: 16,
|
|
73
|
-
encoding: 'pcm',
|
|
74
|
-
},
|
|
75
|
-
};
|
|
76
|
-
eventQueue.push(startEvent);
|
|
77
|
-
});
|
|
78
|
-
ws.on('message', (data) => {
|
|
79
|
-
try {
|
|
80
|
-
const message = JSON.parse(data.toString());
|
|
81
|
-
console.log('[OpenAI Realtime] Received message:', message.type);
|
|
82
|
-
switch (message.type) {
|
|
83
|
-
case 'session.created':
|
|
84
|
-
case 'session.updated':
|
|
85
|
-
console.log('[OpenAI Realtime] Session configured');
|
|
86
|
-
break;
|
|
87
|
-
case 'input_audio_buffer.speech_started': {
|
|
88
|
-
const vadStartEvent = {
|
|
89
|
-
type: 'vad_speech_start',
|
|
90
|
-
timestamp: new Date().toISOString(),
|
|
91
|
-
audio_ms: message.audio
|
|
92
|
-
? parseInt(message.audio)
|
|
93
|
-
: undefined,
|
|
94
|
-
};
|
|
95
|
-
eventQueue.push(vadStartEvent);
|
|
96
|
-
break;
|
|
97
|
-
}
|
|
98
|
-
case 'input_audio_buffer.speech_stopped': {
|
|
99
|
-
const vadEndEvent = {
|
|
100
|
-
type: 'vad_speech_end',
|
|
101
|
-
timestamp: new Date().toISOString(),
|
|
102
|
-
audio_ms: message.audio
|
|
103
|
-
? parseInt(message.audio)
|
|
104
|
-
: undefined,
|
|
105
|
-
};
|
|
106
|
-
eventQueue.push(vadEndEvent);
|
|
107
|
-
break;
|
|
108
|
-
}
|
|
109
|
-
case 'conversation.item.input_audio_transcription.completed':
|
|
110
|
-
if (message.item?.transcript) {
|
|
111
|
-
currentTranscript = message.item.transcript;
|
|
112
|
-
const deltaEvent = {
|
|
113
|
-
type: 'transcription_delta',
|
|
114
|
-
timestamp: new Date().toISOString(),
|
|
115
|
-
delta: message.item.transcript,
|
|
116
|
-
};
|
|
117
|
-
eventQueue.push(deltaEvent);
|
|
118
|
-
}
|
|
119
|
-
break;
|
|
120
|
-
case 'conversation.item.input_audio_transcription.failed': {
|
|
121
|
-
const errorEvent = {
|
|
122
|
-
type: 'error',
|
|
123
|
-
timestamp: new Date().toISOString(),
|
|
124
|
-
error: message.error?.message || 'Transcription failed',
|
|
125
|
-
};
|
|
126
|
-
eventQueue.push(errorEvent);
|
|
127
|
-
break;
|
|
128
|
-
}
|
|
129
|
-
case 'error':
|
|
130
|
-
error = new Error(message.error?.message || 'Unknown error');
|
|
131
|
-
break;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
catch (err) {
|
|
135
|
-
console.error('[OpenAI Realtime] Error parsing message:', err);
|
|
136
|
-
error = err;
|
|
137
|
-
}
|
|
138
|
-
});
|
|
139
|
-
ws.on('error', (err) => {
|
|
140
|
-
console.error('[OpenAI Realtime] WebSocket error:', err);
|
|
141
|
-
error = err;
|
|
142
|
-
});
|
|
143
|
-
ws.on('close', () => {
|
|
144
|
-
console.log('[OpenAI Realtime] WebSocket closed');
|
|
145
|
-
wsConnected = false;
|
|
146
|
-
transcriptionComplete = true;
|
|
147
|
-
});
|
|
148
|
-
while (!wsConnected && !error) {
|
|
149
|
-
await new Promise(resolve => setTimeout(resolve, 10));
|
|
150
|
-
}
|
|
151
|
-
if (error) {
|
|
152
|
-
throw error;
|
|
153
|
-
}
|
|
154
|
-
while (!sessionConfigured && !error) {
|
|
155
|
-
await new Promise(resolve => setTimeout(resolve, 10));
|
|
156
|
-
}
|
|
157
|
-
if (audio instanceof ReadableStream) {
|
|
158
|
-
const reader = audio.getReader();
|
|
159
|
-
(async () => {
|
|
160
|
-
try {
|
|
161
|
-
while (true) {
|
|
162
|
-
const { done, value } = await reader.read();
|
|
163
|
-
if (done) {
|
|
164
|
-
audioStreamEnded = true;
|
|
165
|
-
break;
|
|
166
|
-
}
|
|
167
|
-
if (value && wsConnected) {
|
|
168
|
-
const pcmData = await convertToPCM16(value, config.options?.audio_format);
|
|
169
|
-
const audioMessage = {
|
|
170
|
-
type: 'input_audio_buffer.append',
|
|
171
|
-
audio: Buffer.from(pcmData).toString('base64'),
|
|
172
|
-
};
|
|
173
|
-
ws.send(JSON.stringify(audioMessage));
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
if (wsConnected) {
|
|
177
|
-
ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
catch (err) {
|
|
181
|
-
console.error('[OpenAI Realtime] Error reading audio stream:', err);
|
|
182
|
-
error = err;
|
|
183
|
-
}
|
|
184
|
-
finally {
|
|
185
|
-
reader.releaseLock();
|
|
186
|
-
}
|
|
187
|
-
})();
|
|
188
|
-
}
|
|
189
|
-
else if (audio instanceof ArrayBuffer) {
|
|
190
|
-
const pcmData = await convertToPCM16(new Uint8Array(audio), config.options?.audio_format);
|
|
191
|
-
const audioMessage = {
|
|
192
|
-
type: 'input_audio_buffer.append',
|
|
193
|
-
audio: Buffer.from(pcmData).toString('base64'),
|
|
194
|
-
};
|
|
195
|
-
ws.send(JSON.stringify(audioMessage));
|
|
196
|
-
ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
|
|
197
|
-
audioStreamEnded = true;
|
|
198
|
-
}
|
|
199
|
-
else if (typeof audio === 'string') {
|
|
200
|
-
const audioMessage = {
|
|
201
|
-
type: 'input_audio_buffer.append',
|
|
202
|
-
audio: audio,
|
|
203
|
-
};
|
|
204
|
-
ws.send(JSON.stringify(audioMessage));
|
|
205
|
-
ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
|
|
206
|
-
audioStreamEnded = true;
|
|
207
|
-
}
|
|
208
|
-
while (!transcriptionComplete || eventQueue.length > 0) {
|
|
209
|
-
if (error) {
|
|
210
|
-
throw error;
|
|
211
|
-
}
|
|
212
|
-
if (eventQueue.length > 0) {
|
|
213
|
-
const event = eventQueue.shift();
|
|
214
|
-
yield event;
|
|
215
|
-
}
|
|
216
|
-
else {
|
|
217
|
-
await new Promise(resolve => setTimeout(resolve, 10));
|
|
218
|
-
}
|
|
219
|
-
if (audioStreamEnded && eventQueue.length === 0) {
|
|
220
|
-
await new Promise(resolve => setTimeout(resolve, 500));
|
|
221
|
-
if (currentTranscript) {
|
|
222
|
-
const completeEvent = {
|
|
223
|
-
type: 'transcription_complete',
|
|
224
|
-
timestamp: new Date().toISOString(),
|
|
225
|
-
text: currentTranscript,
|
|
226
|
-
};
|
|
227
|
-
yield completeEvent;
|
|
228
|
-
}
|
|
229
|
-
ws.close();
|
|
230
|
-
break;
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
async function convertToPCM16(audioData, audioFormat) {
|
|
235
|
-
const { AudioConverter } = await import('./audio_converter.js');
|
|
236
|
-
return AudioConverter.convertToPCM16(audioData, audioFormat, 24000);
|
|
237
|
-
}
|
|
238
|
-
//# sourceMappingURL=openai_realtime_client.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"openai_realtime_client.js","sourceRoot":"","sources":["../../utils/openai_realtime_client.ts"],"names":[],"mappings":"AAKA,IAAI,SAAc,CAAC;AACnB,IAAI,OAAO,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;IACpD,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;AACjC,CAAC;KAAM,CAAC;IAEJ,SAAS,GAAG,IAAI,CAAC;AACrB,CAAC;AAgCD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,KAA+B,EAC/B,MAA4B;IAG5B,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YACpC,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACL,MAAM,IAAI,KAAK,CACX,gEAAgE,CACnE,CAAC;QACN,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,kCAAkC,CAAC;IACjD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,oCAAoC,CAAC;IAGnE,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,EAAE;QAC5B,OAAO,EAAE;YACL,aAAa,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;YACxC,aAAa,EAAE,aAAa;SAC/B;KACJ,CAAC,CAAC;IAGH,MAAM,UAAU,GAAyB,EAAE,CAAC;IAC5C,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAClC,IAAI,iBAAiB,GAAG,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAiB,IAAI,CAAC;IAG/B,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACf,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACrD,WAAW,GAAG,IAAI,CAAC;QAGnB,MAAM,aAAa,GAAG;YAClB,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE;gBACL,KAAK,EAAE,KAAK;gBACZ,UAAU,EAAE,CAAC,MAAM,CAAC;gBACpB,YAAY,EAAE,wCAAwC;gBACtD,KAAK,EAAE,MAAM;gBACb,kBAAkB,EAAE,OAAO;gBAC3B,mBAAmB,EAAE,OAAO;gBAC5B,yBAAyB,EAAE;oBACvB,OAAO,EAAE,IAAI;oBACb,KAAK,EAAE,WAAW;iBACrB;gBACD,cAAc,EACV,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,KAAK,KAAK;oBAClC,CAAC,CAAC;wBACI,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,IAAI,YAAY;wBAC/C,SAAS,EAAE,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,SAAS,IAAI,GAAG;wBAChD,iBAAiB,EACb,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,iBAAiB,IAAI,GAAG;wBACjD,mBAAmB,EACf,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,mBAAmB;4BACxC,GAAG;qBACV;oBACH,CAAC,CAAC,IAAI;gBACd,WAAW,EAAE,MAAM,CAAC,OAAO,EAAE,WAAW,IAAI,GAAG;gBAC/C,iBAAiB,EAAE,KAAK;aAC3B;SACJ,CAAC;QAEF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC;QACvC,iBAAiB,GAAG,IAAI,CAAC;QAGzB,MAAM,UAAU,GAA4B;YACxC,IAAI,EAAE,qBAAqB;YAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,IAAI,MAAM;YACjD,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ;YAClC,YAAY,EAAE;gBACV,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,CAAC;gBACX,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE,KAAK;aAClB;SACJ,CAAC;QACF,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;QAC9B,IAAI,CAAC;YACD,MAAM,OAAO,GAA0B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAEjE,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;gBACnB,KAAK,iBAAiB,CAAC;gBACvB,KAAK,iBAAiB;oBAClB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;oBACpD,MAAM;gBAEV,KAAK,mCAAmC,CAAC,CAAC,CAAC;oBACvC,MAAM,aAAa,GAAmB;wBAClC,IAAI,EAAE,kBAAkB;wBACxB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACnC,QAAQ,EAAE,OAAO,CAAC,KAAK;4BACnB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;4BACzB,CAAC,CAAC,SAAS;qBAClB,CAAC;oBACF,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC/B,MAAM;gBACV,CAAC;gBAED,KAAK,mCAAmC,CAAC,CAAC,CAAC;oBACvC,MAAM,WAAW,GAAmB;wBAChC,IAAI,EAAE,gBAAgB;wBACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACnC,QAAQ,EAAE,OAAO,CAAC,KAAK;4BACnB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;4BACzB,CAAC,CAAC,SAAS;qBAClB,CAAC;oBACF,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC7B,MAAM;gBACV,CAAC;gBAED,KAAK,uDAAuD;oBACxD,IAAI,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC;wBAC3B,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;wBAC5C,MAAM,UAAU,GAA4B;4BACxC,IAAI,EAAE,qBAAqB;4BAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BACnC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU;yBACjC,CAAC;wBACF,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAChC,CAAC;oBACD,MAAM;gBAEV,KAAK,oDAAoD,CAAC,CAAC,CAAC;oBACxD,MAAM,UAAU,GAAe;wBAC3B,IAAI,EAAE,OAAO;wBACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACnC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,IAAI,sBAAsB;qBAC1D,CAAC;oBACF,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAC5B,MAAM;gBACV,CAAC;gBAED,KAAK,OAAO;oBACR,KAAK,GAAG,IAAI,KAAK,CACb,OAAO,CAAC,KAAK,EAAE,OAAO,IAAI,eAAe,CAC5C,CAAC;oBACF,MAAM;YACd,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,GAAG,CAAC,CAAC;YAC/D,KAAK,GAAG,GAAY,CAAC;QACzB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;QAC1B,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;QACzD,KAAK,GAAG,GAAG,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAClD,WAAW,GAAG,KAAK,CAAC;QACpB,qBAAqB,GAAG,IAAI,CAAC;IACjC,CAAC,CAAC,CAAC;IAGH,OAAO,CAAC,WAAW,IAAI,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACR,MAAM,KAAK,CAAC;IAChB,CAAC;IAGD,OAAO,CAAC,iBAAiB,IAAI,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAGD,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;QAElC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QAEjC,CAAC,KAAK,IAAI,EAAE;YACR,IAAI,CAAC;gBACD,OAAO,IAAI,EAAE,CAAC;oBACV,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAE5C,IAAI,IAAI,EAAE,CAAC;wBACP,gBAAgB,GAAG,IAAI,CAAC;wBACxB,MAAM;oBACV,CAAC;oBAED,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;wBAEvB,MAAM,OAAO,GAAG,MAAM,cAAc,CAChC,KAAK,EACL,MAAM,CAAC,OAAO,EAAE,YAAY,CAC/B,CAAC;wBAGF,MAAM,YAAY,GAAG;4BACjB,IAAI,EAAE,2BAA2B;4BACjC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;yBACjD,CAAC;wBACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC;oBAC1C,CAAC;gBACL,CAAC;gBAGD,IAAI,WAAW,EAAE,CAAC;oBACd,EAAE,CAAC,IAAI,CACH,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CACxD,CAAC;gBACN,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CACT,+CAA+C,EAC/C,GAAG,CACN,CAAC;gBACF,KAAK,GAAG,GAAY,CAAC;YACzB,CAAC;oBAAS,CAAC;gBACP,MAAM,CAAC,WAAW,EAAE,CAAC;YACzB,CAAC;QACL,CAAC,CAAC,EAAE,CAAC;IACT,CAAC;SAAM,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;QAEtC,MAAM,OAAO,GAAG,MAAM,cAAc,CAChC,IAAI,UAAU,CAAC,KAAK,CAAC,EACrB,MAAM,CAAC,OAAO,EAAE,YAAY,CAC/B,CAAC;QAEF,MAAM,YAAY,GAAG;YACjB,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;SACjD,CAAC;QACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC;QACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC,CAAC;QAC/D,gBAAgB,GAAG,IAAI,CAAC;IAC5B,CAAC;SAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAEnC,MAAM,YAAY,GAAG;YACjB,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,KAAK;SACf,CAAC;QACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC;QACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC,CAAC;QAC/D,gBAAgB,GAAG,IAAI,CAAC;IAC5B,CAAC;IAGD,OAAO,CAAC,qBAAqB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrD,IAAI,KAAK,EAAE,CAAC;YACR,MAAM,KAAK,CAAC;QAChB,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,EAAG,CAAC;YAClC,MAAM,KAAK,CAAC;QAChB,CAAC;aAAM,CAAC;YAEJ,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;QAGD,IAAI,gBAAgB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAE9C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAGvD,IAAI,iBAAiB,EAAE,CAAC;gBACpB,MAAM,aAAa,GAA+B;oBAC9C,IAAI,EAAE,wBAAwB;oBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,IAAI,EAAE,iBAAiB;iBAC1B,CAAC;gBACF,MAAM,aAAa,CAAC;YACxB,CAAC;YAGD,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,MAAM;QACV,CAAC;IACL,CAAC;AACL,CAAC;AAKD,KAAK,UAAU,cAAc,CACzB,SAAqB,EACrB,WAA+C;IAG/C,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAGhE,OAAO,cAAc,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;AACxE,CAAC"}
|