@framers/agentos 0.1.186 → 0.1.187

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ /**
2
+ * @module voice-pipeline/providers/OpenAIRealtimeTTS
3
+ *
4
+ * Streaming TTS via OpenAI's Realtime API (gpt-4o-mini-tts).
5
+ * Implements {@link IStreamingTTS} with the same interface as ElevenLabsStreamingTTS.
6
+ *
7
+ * Protocol: WebSocket to wss://api.openai.com/v1/realtime with session-based events.
8
+ * Text is sent via conversation.item.create + response.create events.
9
+ * Audio arrives as response.audio.delta base64-encoded chunks.
10
+ */
11
+ import type { IStreamingTTS, StreamingTTSSession, StreamingTTSConfig } from '../types.js';
12
+ export interface OpenAIRealtimeTTSConfig {
13
+ apiKey: string;
14
+ model?: string;
15
+ baseUrl?: string;
16
+ }
17
+ export declare class OpenAIRealtimeTTS implements IStreamingTTS {
18
+ readonly providerId = "openai-realtime";
19
+ private readonly config;
20
+ constructor(config: OpenAIRealtimeTTSConfig);
21
+ startSession(config?: StreamingTTSConfig): Promise<StreamingTTSSession>;
22
+ }
23
+ //# sourceMappingURL=OpenAIRealtimeTTS.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"OpenAIRealtimeTTS.d.ts","sourceRoot":"","sources":["../../../src/voice-pipeline/providers/OpenAIRealtimeTTS.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EACV,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAEnB,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAiID,qBAAa,iBAAkB,YAAW,aAAa;IACrD,QAAQ,CAAC,UAAU,qBAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;gBAErC,MAAM,EAAE,uBAAuB;IAIrC,YAAY,CAAC,MAAM,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAK9E"}
@@ -0,0 +1,137 @@
1
+ /**
2
+ * @module voice-pipeline/providers/OpenAIRealtimeTTS
3
+ *
4
+ * Streaming TTS via OpenAI's Realtime API (gpt-4o-mini-tts).
5
+ * Implements {@link IStreamingTTS} with the same interface as ElevenLabsStreamingTTS.
6
+ *
7
+ * Protocol: WebSocket to wss://api.openai.com/v1/realtime with session-based events.
8
+ * Text is sent via conversation.item.create + response.create events.
9
+ * Audio arrives as response.audio.delta base64-encoded chunks.
10
+ */
11
+ import { EventEmitter } from 'node:events';
12
+ import WebSocket from 'ws';
13
+ class OpenAIRealtimeTTSSession extends EventEmitter {
14
+ constructor(config, sessionConfig) {
15
+ super();
16
+ this.config = config;
17
+ this.sessionConfig = sessionConfig;
18
+ this.ws = null;
19
+ this.closed = false;
20
+ this.pendingFlush = false;
21
+ /** Accumulates text pushed via pushTokens so we can attach it to audio chunks. */
22
+ this.pendingText = '';
23
+ }
24
+ async connect() {
25
+ const model = this.config.model ?? 'gpt-4o-mini-tts';
26
+ const baseUrl = this.config.baseUrl ?? 'wss://api.openai.com/v1/realtime';
27
+ const url = `${baseUrl}?model=${model}`;
28
+ return new Promise((resolve, reject) => {
29
+ this.ws = new WebSocket(url, {
30
+ headers: {
31
+ Authorization: `Bearer ${this.config.apiKey}`,
32
+ 'OpenAI-Beta': 'realtime=v1',
33
+ },
34
+ });
35
+ this.ws.on('open', () => {
36
+ this.ws.send(JSON.stringify({
37
+ type: 'session.update',
38
+ session: {
39
+ modalities: ['audio', 'text'],
40
+ voice: this.sessionConfig.voice ?? 'alloy',
41
+ },
42
+ }));
43
+ resolve();
44
+ });
45
+ this.ws.on('error', (err) => {
46
+ this.emit('error', err);
47
+ reject(err);
48
+ });
49
+ this.ws.on('message', (data) => {
50
+ const msg = JSON.parse(typeof data === 'string' ? data : data.toString('utf-8'));
51
+ this._handleMessage(msg);
52
+ });
53
+ this.ws.on('close', () => {
54
+ this.closed = true;
55
+ this.emit('close');
56
+ });
57
+ });
58
+ }
59
+ pushTokens(tokens) {
60
+ if (this.closed || !this.ws || this.ws.readyState !== WebSocket.OPEN)
61
+ return;
62
+ this.pendingText += tokens;
63
+ this.ws.send(JSON.stringify({
64
+ type: 'conversation.item.create',
65
+ item: {
66
+ type: 'message',
67
+ role: 'user',
68
+ content: [{ type: 'input_text', text: tokens }],
69
+ },
70
+ }));
71
+ }
72
+ async flush() {
73
+ if (this.closed || !this.ws || this.ws.readyState !== WebSocket.OPEN)
74
+ return;
75
+ this.pendingFlush = true;
76
+ this.ws.send(JSON.stringify({ type: 'response.create' }));
77
+ return new Promise((resolve) => {
78
+ const onDone = () => {
79
+ this.removeListener('_internal_flush', onDone);
80
+ resolve();
81
+ };
82
+ this.on('_internal_flush', onDone);
83
+ setTimeout(() => {
84
+ this.removeListener('_internal_flush', onDone);
85
+ resolve();
86
+ }, 30000);
87
+ });
88
+ }
89
+ cancel() {
90
+ if (this.ws && this.ws.readyState === WebSocket.OPEN) {
91
+ this.ws.send(JSON.stringify({ type: 'response.cancel' }));
92
+ }
93
+ }
94
+ close() {
95
+ this.closed = true;
96
+ this.ws?.close();
97
+ }
98
+ _handleMessage(msg) {
99
+ if (msg.type === 'response.audio.delta') {
100
+ const audioB64 = msg.delta;
101
+ if (audioB64) {
102
+ const audioBuffer = Buffer.from(audioB64, 'base64');
103
+ const chunk = {
104
+ audio: audioBuffer,
105
+ format: 'pcm',
106
+ sampleRate: 24000,
107
+ durationMs: Math.round((audioBuffer.byteLength / (24000 * 2)) * 1000),
108
+ text: this.pendingText,
109
+ };
110
+ this.emit('audio', chunk);
111
+ }
112
+ }
113
+ else if (msg.type === 'response.audio.done' || msg.type === 'response.done') {
114
+ if (this.pendingFlush) {
115
+ this.pendingFlush = false;
116
+ this.pendingText = '';
117
+ this.emit('_internal_flush');
118
+ this.emit('flush_complete');
119
+ }
120
+ }
121
+ else if (msg.type === 'error') {
122
+ this.emit('error', new Error(String(msg.error?.message ?? 'Unknown error')));
123
+ }
124
+ }
125
+ }
126
+ export class OpenAIRealtimeTTS {
127
+ constructor(config) {
128
+ this.providerId = 'openai-realtime';
129
+ this.config = config;
130
+ }
131
+ async startSession(config) {
132
+ const session = new OpenAIRealtimeTTSSession(this.config, config ?? {});
133
+ await session.connect();
134
+ return session;
135
+ }
136
+ }
137
+ //# sourceMappingURL=OpenAIRealtimeTTS.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"OpenAIRealtimeTTS.js","sourceRoot":"","sources":["../../../src/voice-pipeline/providers/OpenAIRealtimeTTS.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,SAAS,MAAM,IAAI,CAAC;AAc3B,MAAM,wBAAyB,SAAQ,YAAY;IAOjD,YACmB,MAA+B,EAC/B,aAAiC;QAElD,KAAK,EAAE,CAAC;QAHS,WAAM,GAAN,MAAM,CAAyB;QAC/B,kBAAa,GAAb,aAAa,CAAoB;QAR5C,OAAE,GAAqB,IAAI,CAAC;QAC5B,WAAM,GAAG,KAAK,CAAC;QACf,iBAAY,GAAG,KAAK,CAAC;QAC7B,kFAAkF;QAC1E,gBAAW,GAAG,EAAE,CAAC;IAOzB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,iBAAiB,CAAC;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,kCAAkC,CAAC;QAC1E,MAAM,GAAG,GAAG,GAAG,OAAO,UAAU,KAAK,EAAE,CAAC;QAExC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;gBAC3B,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;oBAC7C,aAAa,EAAE,aAAa;iBAC7B;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,EAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBAC3B,IAAI,EAAE,gBAAgB;oBACtB,OAAO,EAAE;wBACP,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;wBAC7B,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,IAAI,OAAO;qBAC3C;iBACF,CAAC,CAAC,CAAC;gBACJ,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAqB,EAAE,EAAE;gBAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBACjF,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACvB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBACnB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,UAAU,CAAC,MAAc;QACvB,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;YAAE,OAAO;QAE7E,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;QAE3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAC1B,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACJ,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;aAChD;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;YAAE,OAAO;QAE7E,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;QAE1D,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,GAAG,EAAE;gBAClB,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;gBAC/C,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;YACnC,UAAU,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;gBAC/C,OAAO,EAAE,CAAC;YACZ,CAAC,EAAE,KAAM,CAAC,CAAC;QACb,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM;QACJ,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACrD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,GAAG,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAI,GAAW,CAAC,KAAe,CAAC;YAC9C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACpD,MAAM,KAAK,GAAsB;oBAC/B,KAAK,EAAE,WAAW;oBAClB,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,UAAU,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;oBACrE,IAAI,EAAE,IAAI,CAAC,WAAW;iBACvB,CAAC;gBACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,IAAI,GAAG,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YAC9E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;gBAC1B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,MAAM,CAAE,GAAW,CAAC,KAAK,EAAE,OAAO,IAAI,eAAe,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,iBAAiB;IAI5B,YAAY,MAA+B;QAHlC,eAAU,GAAG,iBAAiB,CAAC;QAItC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAA2B;QAC5C,MAAM,OAAO,GAAG,IAAI,wBAAwB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QACxE,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
@@ -16,4 +16,5 @@ export { AgentSessionVoiceAdapter } from './AgentSessionVoiceAdapter.js';
16
16
  export { OpenAIBatchTTS, type OpenAIBatchTTSConfig } from './OpenAIBatchTTS.js';
17
17
  export { ElevenLabsBatchTTS, type ElevenLabsBatchTTSConfig } from './ElevenLabsBatchTTS.js';
18
18
  export { BatchTTSFallback } from './BatchTTSFallback.js';
19
+ export { OpenAIRealtimeTTS, type OpenAIRealtimeTTSConfig } from './OpenAIRealtimeTTS.js';
19
20
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/voice-pipeline/providers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,oBAAoB,EAAE,KAAK,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AAClG,OAAO,EACL,sBAAsB,EACtB,KAAK,4BAA4B,GAClC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,sBAAsB,EACtB,KAAK,4BAA4B,GAClC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,kBAAkB,EAAE,KAAK,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/voice-pipeline/providers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,oBAAoB,EAAE,KAAK,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AAClG,OAAO,EACL,sBAAsB,EACtB,KAAK,4BAA4B,GAClC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,sBAAsB,EACtB,KAAK,4BAA4B,GAClC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,kBAAkB,EAAE,KAAK,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC"}
@@ -16,4 +16,5 @@ export { AgentSessionVoiceAdapter } from './AgentSessionVoiceAdapter.js';
16
16
  export { OpenAIBatchTTS } from './OpenAIBatchTTS.js';
17
17
  export { ElevenLabsBatchTTS } from './ElevenLabsBatchTTS.js';
18
18
  export { BatchTTSFallback } from './BatchTTSFallback.js';
19
+ export { OpenAIRealtimeTTS } from './OpenAIRealtimeTTS.js';
19
20
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/voice-pipeline/providers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,oBAAoB,EAAmC,MAAM,2BAA2B,CAAC;AAClG,OAAO,EACL,sBAAsB,GAEvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,sBAAsB,GAEvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,cAAc,EAA6B,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,kBAAkB,EAAiC,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/voice-pipeline/providers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,oBAAoB,EAAmC,MAAM,2BAA2B,CAAC;AAClG,OAAO,EACL,sBAAsB,GAEvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,sBAAsB,GAEvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,cAAc,EAA6B,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,kBAAkB,EAAiC,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAgC,MAAM,wBAAwB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@framers/agentos",
3
- "version": "0.1.186",
3
+ "version": "0.1.187",
4
4
  "description": "Modular AgentOS orchestration library",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",