@cogitator-ai/voice 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +6 -0
- package/dist/audio.js.map +1 -1
- package/dist/pipeline/pipeline-session.d.ts +2 -1
- package/dist/pipeline/pipeline-session.d.ts.map +1 -1
- package/dist/pipeline/pipeline-session.js +8 -4
- package/dist/pipeline/pipeline-session.js.map +1 -1
- package/dist/realtime/gemini-realtime.d.ts +8 -0
- package/dist/realtime/gemini-realtime.d.ts.map +1 -1
- package/dist/realtime/gemini-realtime.js +84 -22
- package/dist/realtime/gemini-realtime.js.map +1 -1
- package/dist/realtime/openai-realtime.d.ts +2 -0
- package/dist/realtime/openai-realtime.d.ts.map +1 -1
- package/dist/realtime/openai-realtime.js +72 -14
- package/dist/realtime/openai-realtime.js.map +1 -1
- package/dist/realtime/realtime-session.d.ts +4 -0
- package/dist/realtime/realtime-session.d.ts.map +1 -1
- package/dist/realtime/realtime-session.js +15 -2
- package/dist/realtime/realtime-session.js.map +1 -1
- package/dist/stt/deepgram-stt.d.ts.map +1 -1
- package/dist/stt/deepgram-stt.js +39 -9
- package/dist/stt/deepgram-stt.js.map +1 -1
- package/dist/stt/openai-stt.js +8 -0
- package/dist/stt/openai-stt.js.map +1 -1
- package/dist/tools.d.ts +8 -0
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +12 -4
- package/dist/tools.js.map +1 -1
- package/dist/transport/index.d.ts +1 -0
- package/dist/transport/index.d.ts.map +1 -1
- package/dist/transport/ws-transport.d.ts +16 -1
- package/dist/transport/ws-transport.d.ts.map +1 -1
- package/dist/transport/ws-transport.js +74 -8
- package/dist/transport/ws-transport.js.map +1 -1
- package/dist/tts/elevenlabs-tts.d.ts +1 -0
- package/dist/tts/elevenlabs-tts.d.ts.map +1 -1
- package/dist/tts/elevenlabs-tts.js +11 -2
- package/dist/tts/elevenlabs-tts.js.map +1 -1
- package/dist/tts/openai-tts.d.ts +1 -0
- package/dist/tts/openai-tts.d.ts.map +1 -1
- package/dist/tts/openai-tts.js +23 -9
- package/dist/tts/openai-tts.js.map +1 -1
- package/dist/types.d.ts +1 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vad/silero-vad.d.ts +2 -0
- package/dist/vad/silero-vad.d.ts.map +1 -1
- package/dist/vad/silero-vad.js +13 -0
- package/dist/vad/silero-vad.js.map +1 -1
- package/dist/voice-agent.js +8 -8
- package/dist/voice-agent.js.map +1 -1
- package/package.json +15 -15
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Cogitator Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/dist/audio.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAQ5D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAQ3D;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAQ,GAAG,MAAM,CAuBhE;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,
|
|
1
|
+
{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAQ5D;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAQ3D;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAQ,GAAG,MAAM,CAuBhE;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAiCnF;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAiB9F;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAK1D"}
|
package/dist/audio.js
CHANGED
|
@@ -55,6 +55,9 @@ export function wavToPcm(wav) {
|
|
|
55
55
|
while (offset + 8 <= wav.length) {
|
|
56
56
|
const chunkId = wav.toString('ascii', offset, offset + 4);
|
|
57
57
|
const chunkSize = view.getUint32(offset + 4, true);
|
|
58
|
+
if (offset + 8 + chunkSize > wav.length) {
|
|
59
|
+
throw new Error('Truncated WAV chunk');
|
|
60
|
+
}
|
|
58
61
|
if (chunkId === 'data') {
|
|
59
62
|
const pcmData = wav.subarray(offset + 8, offset + 8 + chunkSize);
|
|
60
63
|
return { samples: pcm16ToFloat32(pcmData), sampleRate };
|
|
@@ -64,6 +67,9 @@ export function wavToPcm(wav) {
|
|
|
64
67
|
throw new Error('WAV file missing data chunk');
|
|
65
68
|
}
|
|
66
69
|
export function resample(samples, fromRate, toRate) {
|
|
70
|
+
if (fromRate <= 0 || toRate <= 0) {
|
|
71
|
+
throw new Error(`Sample rates must be positive, got fromRate=${fromRate}, toRate=${toRate}`);
|
|
72
|
+
}
|
|
67
73
|
if (fromRate === toRate)
|
|
68
74
|
return new Float32Array(samples);
|
|
69
75
|
const ratio = fromRate / toRate;
|
package/dist/audio.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audio.js","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACvC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC;IACrD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,UAAU,GAAG,KAAK;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,CAAC,CAAC;IACnB,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,MAAM,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,QAAQ,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;IAElD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxB,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAErC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,IACE,GAAG,CAAC,MAAM,GAAG,EAAE;QACf,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM;QACtC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,EACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IACtE,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,aAAa,KAAK,EAAE,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,2BAA2B,WAAW,MAAM,aAAa,4BAA4B,CACtF,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAE5C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,OAAO,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;QAC1D,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IAC1B,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAqB,EAAE,QAAgB,EAAE,MAAc;IAC9E,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,GAAG,KAAK,CAAC;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAqB;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"audio.js","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACvC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC;IACrD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,UAAU,GAAG,KAAK;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,CAAC,CAAC;IACnB,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,MAAM,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,QAAQ,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;IAElD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxB,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAErC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,IACE,GAAG,CAAC,MAAM,GAAG,EAAE;QACf,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM;QACtC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,EACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IACtE,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,aAAa,KAAK,EAAE,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,2BAA2B,WAAW,MAAM,aAAa,4BAA4B,CACtF,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAE5C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,OAAO,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACnD,IAAI,MAAM,GAAG,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;QAC1D,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC;IAC1B,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAqB,EAAE,QAAgB,EAAE,MAAc;IAC9E,IAAI,QAAQ,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,YAAY,MAAM,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,GAAG,KAAK,CAAC;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAqB;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -18,12 +18,13 @@ export declare class PipelineSession extends EventEmitter<PipelineSessionEvents>
|
|
|
18
18
|
private interrupted;
|
|
19
19
|
private closed;
|
|
20
20
|
private activeProcessing;
|
|
21
|
+
private vadQueue;
|
|
21
22
|
constructor(config: VoicePipelineConfig);
|
|
22
23
|
pushAudio(chunk: Buffer): void;
|
|
23
24
|
endAudio(): void;
|
|
24
25
|
interrupt(): void;
|
|
25
26
|
close(): Promise<void>;
|
|
26
|
-
private
|
|
27
|
+
private applyVADEvent;
|
|
27
28
|
private handleNoVAD;
|
|
28
29
|
private openStream;
|
|
29
30
|
private finishListening;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-session.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"pipeline-session.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAuB,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAK5E,UAAU,qBAAqB;IAC7B,YAAY,EAAE,EAAE,CAAC;IACjB,UAAU,EAAE,EAAE,CAAC;IACf,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7C,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,qBAAa,eAAgB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IACtE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,QAAQ,CAAoC;gBAExC,MAAM,EAAE,mBAAmB;IAQvC,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAe9B,QAAQ,IAAI,IAAI;IAMhB,SAAS,IAAI,IAAI;IAaX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB5B,OAAO,CAAC,aAAa;IA0BrB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,UAAU;IAgBlB,OAAO,CAAC,eAAe;CAiDxB"}
|
|
@@ -10,6 +10,7 @@ export class PipelineSession extends EventEmitter {
|
|
|
10
10
|
interrupted = false;
|
|
11
11
|
closed = false;
|
|
12
12
|
activeProcessing = null;
|
|
13
|
+
vadQueue = Promise.resolve();
|
|
13
14
|
constructor(config) {
|
|
14
15
|
super();
|
|
15
16
|
this.stt = config.stt;
|
|
@@ -21,7 +22,12 @@ export class PipelineSession extends EventEmitter {
|
|
|
21
22
|
if (this.closed)
|
|
22
23
|
return;
|
|
23
24
|
if (this.vad) {
|
|
24
|
-
|
|
25
|
+
const samples = pcm16ToFloat32(chunk);
|
|
26
|
+
const eventOrPromise = this.vad.process(samples);
|
|
27
|
+
this.vadQueue = this.vadQueue.then(async () => {
|
|
28
|
+
const event = await eventOrPromise;
|
|
29
|
+
this.applyVADEvent(event, chunk);
|
|
30
|
+
});
|
|
25
31
|
}
|
|
26
32
|
else {
|
|
27
33
|
this.handleNoVAD(chunk);
|
|
@@ -55,9 +61,7 @@ export class PipelineSession extends EventEmitter {
|
|
|
55
61
|
}
|
|
56
62
|
this.removeAllListeners();
|
|
57
63
|
}
|
|
58
|
-
|
|
59
|
-
const samples = pcm16ToFloat32(chunk);
|
|
60
|
-
const event = await this.vad.process(samples);
|
|
64
|
+
applyVADEvent(event, chunk) {
|
|
61
65
|
switch (event.type) {
|
|
62
66
|
case 'speech_start':
|
|
63
67
|
if (this.state === 'idle') {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-session.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAa7C,MAAM,OAAO,eAAgB,SAAQ,YAAmC;IACrD,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,KAAK,CAA+B;IAC7C,KAAK,GAAiB,MAAM,CAAC;IAC7B,MAAM,GAAqB,IAAI,CAAC;IAChC,WAAW,GAAG,KAAK,CAAC;IACpB,MAAM,GAAG,KAAK,CAAC;IACf,gBAAgB,GAAyB,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline-session.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAa7C,MAAM,OAAO,eAAgB,SAAQ,YAAmC;IACrD,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,KAAK,CAA+B;IAC7C,KAAK,GAAiB,MAAM,CAAC;IAC7B,MAAM,GAAqB,IAAI,CAAC;IAChC,WAAW,GAAG,KAAK,CAAC;IACpB,MAAM,GAAG,KAAK,CAAC;IACf,gBAAgB,GAAyB,IAAI,CAAC;IAC9C,QAAQ,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,YAAY,MAA2B;QACrC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAExB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5C,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC;gBACnC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,SAAS;QACP,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAEO,aAAa,CAAC,KAAe,EAAE,KAAa;QAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;oBAC1B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;oBACzB,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM;YAER,KAAK,YAAY;gBACf,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACxB,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9B,CAAC;gBACD,MAAM;YAER,KAAK,QAAQ;gBACX,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3B,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa;QAC/B,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;YACzB,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,UAAU;QAChB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAEtC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;YACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,MAAwB,EAAE,EAAE;YACnD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YACvC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAE3C,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAE1B,MAAM,UAAU,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;gBAC3C,aAAa,CAAC,kBAAkB,EAAE,CAAC;gBAEnC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBACpC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBACzB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,OAAO;gBACT,CAAC;gBAED,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;gBAErC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBACpC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBACzB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;gBACxB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAErD,IAAI,KAAK,EAAE,MAAM,UAAU,IAAI,SAAS,EAAE,CAAC;oBACzC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM;wBAAE,MAAM;oBAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;gBACjC,CAAC;gBAED,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBACzB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACtB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;gBACpB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;QAEL,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
|
|
@@ -2,20 +2,28 @@ import { EventEmitter } from 'node:events';
|
|
|
2
2
|
import type { RealtimeSessionConfig } from '../types.js';
|
|
3
3
|
interface GeminiRealtimeEvents {
|
|
4
4
|
connected: [];
|
|
5
|
+
disconnected: [code: number, reason: string];
|
|
5
6
|
audio: [chunk: Buffer];
|
|
6
7
|
transcript: [text: string, role: 'user' | 'assistant'];
|
|
7
8
|
tool_call: [name: string, args: unknown];
|
|
8
9
|
speech_start: [];
|
|
10
|
+
turn_end: [];
|
|
9
11
|
error: [error: Error];
|
|
10
12
|
}
|
|
11
13
|
export declare class GeminiRealtimeAdapter extends EventEmitter<GeminiRealtimeEvents> {
|
|
12
14
|
private readonly config;
|
|
13
15
|
private readonly model;
|
|
14
16
|
private ws;
|
|
17
|
+
private interrupting;
|
|
15
18
|
constructor(config: RealtimeSessionConfig);
|
|
16
19
|
connect(): Promise<void>;
|
|
17
20
|
pushAudio(chunk: Buffer): void;
|
|
18
21
|
sendText(text: string): void;
|
|
22
|
+
/**
|
|
23
|
+
* Gemini Live barge-in is driven by incoming user audio, not a control message.
|
|
24
|
+
* This sets an internal flag that drops inbound model audio until the current
|
|
25
|
+
* turn completes (turnComplete), simulating an interruption on the consumer side.
|
|
26
|
+
*/
|
|
19
27
|
interrupt(): void;
|
|
20
28
|
close(): void;
|
|
21
29
|
private sendSetup;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAOzD,UAAU,oBAAoB;IAC5B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AA+BD,qBAAa,qBAAsB,SAAQ,YAAY,CAAC,oBAAoB,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,YAAY,CAAS;gBAEjB,MAAM,EAAE,qBAAqB;IAMnC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAmF9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAa9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAS5B;;;;OAIG;IACH,SAAS,IAAI,IAAI;IAIjB,KAAK,IAAI,IAAI;IAQb,OAAO,CAAC,SAAS;IAoCjB,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,mBAAmB;YAkBb,eAAe;IAoC7B,OAAO,CAAC,IAAI;CAMb"}
|
|
@@ -2,24 +2,56 @@ import { EventEmitter } from 'node:events';
|
|
|
2
2
|
import { WebSocket } from 'ws';
|
|
3
3
|
const DEFAULT_MODEL = 'gemini-live-2.5-flash-native-audio';
|
|
4
4
|
const BASE_URL = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
|
|
5
|
+
const CONNECT_TIMEOUT_MS = 30_000;
|
|
5
6
|
export class GeminiRealtimeAdapter extends EventEmitter {
|
|
6
7
|
config;
|
|
7
8
|
model;
|
|
8
9
|
ws = null;
|
|
10
|
+
interrupting = false;
|
|
9
11
|
constructor(config) {
|
|
10
12
|
super();
|
|
11
13
|
this.config = config;
|
|
12
14
|
this.model = config.model ?? DEFAULT_MODEL;
|
|
13
15
|
}
|
|
14
16
|
async connect() {
|
|
17
|
+
if (this.ws) {
|
|
18
|
+
throw new Error('Already connected or connecting — call close() first');
|
|
19
|
+
}
|
|
15
20
|
const url = `${BASE_URL}?key=${this.config.apiKey}`;
|
|
16
21
|
return new Promise((resolve, reject) => {
|
|
17
|
-
|
|
18
|
-
|
|
22
|
+
let settled = false;
|
|
23
|
+
const settleResolve = () => {
|
|
24
|
+
if (settled)
|
|
25
|
+
return;
|
|
26
|
+
settled = true;
|
|
27
|
+
clearTimeout(timer);
|
|
28
|
+
resolve();
|
|
29
|
+
};
|
|
30
|
+
const settleReject = (err) => {
|
|
31
|
+
if (settled)
|
|
32
|
+
return;
|
|
33
|
+
settled = true;
|
|
34
|
+
clearTimeout(timer);
|
|
35
|
+
reject(err);
|
|
36
|
+
};
|
|
37
|
+
const ws = new WebSocket(url);
|
|
38
|
+
this.ws = ws;
|
|
39
|
+
const timer = setTimeout(() => {
|
|
40
|
+
this.ws = null;
|
|
41
|
+
ws.removeAllListeners();
|
|
42
|
+
ws.close();
|
|
43
|
+
settleReject(new Error(`Connect timed out after ${CONNECT_TIMEOUT_MS}ms`));
|
|
44
|
+
}, CONNECT_TIMEOUT_MS);
|
|
45
|
+
ws.on('open', () => {
|
|
19
46
|
this.sendSetup();
|
|
20
47
|
});
|
|
21
|
-
|
|
22
|
-
const
|
|
48
|
+
ws.on('message', (data) => {
|
|
49
|
+
const buf = Array.isArray(data)
|
|
50
|
+
? Buffer.concat(data)
|
|
51
|
+
: data instanceof ArrayBuffer
|
|
52
|
+
? Buffer.from(new Uint8Array(data))
|
|
53
|
+
: data;
|
|
54
|
+
const raw = buf.toString();
|
|
23
55
|
let msg;
|
|
24
56
|
try {
|
|
25
57
|
msg = JSON.parse(raw);
|
|
@@ -30,14 +62,27 @@ export class GeminiRealtimeAdapter extends EventEmitter {
|
|
|
30
62
|
}
|
|
31
63
|
if (msg.setupComplete) {
|
|
32
64
|
this.emit('connected');
|
|
33
|
-
|
|
65
|
+
settleResolve();
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (msg.error) {
|
|
69
|
+
const message = msg.error.message ?? `Gemini error (code: ${msg.error.code ?? 'unknown'})`;
|
|
70
|
+
const err = new Error(message);
|
|
71
|
+
this.emit('error', err);
|
|
72
|
+
settleReject(err);
|
|
34
73
|
return;
|
|
35
74
|
}
|
|
36
75
|
this.handleMessage(msg);
|
|
37
76
|
});
|
|
38
|
-
|
|
77
|
+
ws.on('error', (err) => {
|
|
39
78
|
this.emit('error', err);
|
|
40
|
-
|
|
79
|
+
settleReject(err);
|
|
80
|
+
});
|
|
81
|
+
ws.on('close', (code, reason) => {
|
|
82
|
+
const reasonStr = reason.toString() || 'connection closed';
|
|
83
|
+
this.ws = null;
|
|
84
|
+
this.emit('disconnected', code, reasonStr);
|
|
85
|
+
settleReject(new Error(`WebSocket closed before connect (code ${code}): ${reasonStr}`));
|
|
41
86
|
});
|
|
42
87
|
});
|
|
43
88
|
}
|
|
@@ -61,14 +106,21 @@ export class GeminiRealtimeAdapter extends EventEmitter {
|
|
|
61
106
|
},
|
|
62
107
|
});
|
|
63
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Gemini Live barge-in is driven by incoming user audio, not a control message.
|
|
111
|
+
* This sets an internal flag that drops inbound model audio until the current
|
|
112
|
+
* turn completes (turnComplete), simulating an interruption on the consumer side.
|
|
113
|
+
*/
|
|
64
114
|
interrupt() {
|
|
65
|
-
this.
|
|
66
|
-
clientContent: { turnComplete: true },
|
|
67
|
-
});
|
|
115
|
+
this.interrupting = true;
|
|
68
116
|
}
|
|
69
117
|
close() {
|
|
70
|
-
this.ws
|
|
118
|
+
const ws = this.ws;
|
|
119
|
+
if (!ws)
|
|
120
|
+
return;
|
|
71
121
|
this.ws = null;
|
|
122
|
+
ws.removeAllListeners();
|
|
123
|
+
ws.close();
|
|
72
124
|
}
|
|
73
125
|
sendSetup() {
|
|
74
126
|
const setup = {
|
|
@@ -103,6 +155,11 @@ export class GeminiRealtimeAdapter extends EventEmitter {
|
|
|
103
155
|
this.send({ setup });
|
|
104
156
|
}
|
|
105
157
|
handleMessage(msg) {
|
|
158
|
+
if (msg.error) {
|
|
159
|
+
const message = msg.error.message ?? `Gemini error (code: ${msg.error.code ?? 'unknown'})`;
|
|
160
|
+
this.emit('error', new Error(message));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
106
163
|
if (msg.serverContent) {
|
|
107
164
|
this.handleServerContent(msg.serverContent);
|
|
108
165
|
}
|
|
@@ -111,24 +168,29 @@ export class GeminiRealtimeAdapter extends EventEmitter {
|
|
|
111
168
|
}
|
|
112
169
|
}
|
|
113
170
|
handleServerContent(content) {
|
|
114
|
-
if (!content.modelTurn?.parts)
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
171
|
+
if (!this.interrupting && content.modelTurn?.parts) {
|
|
172
|
+
for (const part of content.modelTurn.parts) {
|
|
173
|
+
if (part.inlineData) {
|
|
174
|
+
this.emit('audio', Buffer.from(part.inlineData.data, 'base64'));
|
|
175
|
+
}
|
|
176
|
+
if (part.text) {
|
|
177
|
+
this.emit('transcript', part.text, 'assistant');
|
|
178
|
+
}
|
|
122
179
|
}
|
|
123
180
|
}
|
|
181
|
+
if (content.turnComplete) {
|
|
182
|
+
this.interrupting = false;
|
|
183
|
+
this.emit('turn_end');
|
|
184
|
+
}
|
|
124
185
|
}
|
|
125
186
|
async handleToolCalls(calls) {
|
|
126
|
-
const responses = await Promise.all(calls.map(async (call) => {
|
|
187
|
+
const responses = await Promise.all(calls.map(async (call, index) => {
|
|
188
|
+
const id = call.id ?? `${call.name}-${index}`;
|
|
127
189
|
this.emit('tool_call', call.name, call.args);
|
|
128
190
|
const tool = this.config.tools?.find((t) => t.name === call.name);
|
|
129
191
|
if (!tool) {
|
|
130
192
|
return {
|
|
131
|
-
id
|
|
193
|
+
id,
|
|
132
194
|
name: call.name,
|
|
133
195
|
response: { result: JSON.stringify({ error: `Unknown tool: ${call.name}` }) },
|
|
134
196
|
};
|
|
@@ -142,7 +204,7 @@ export class GeminiRealtimeAdapter extends EventEmitter {
|
|
|
142
204
|
result = JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
|
|
143
205
|
}
|
|
144
206
|
return {
|
|
145
|
-
id
|
|
207
|
+
id,
|
|
146
208
|
name: call.name,
|
|
147
209
|
response: { result },
|
|
148
210
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini-realtime.js","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,oCAAoC,CAAC;AAC3D,MAAM,QAAQ,GACZ,sHAAsH,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini-realtime.js","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,oCAAoC,CAAC;AAC3D,MAAM,QAAQ,GACZ,sHAAsH,CAAC;AACzH,MAAM,kBAAkB,GAAG,MAAM,CAAC;AA0ClC,MAAM,OAAO,qBAAsB,SAAQ,YAAkC;IAC1D,MAAM,CAAwB;IAC9B,KAAK,CAAS;IACvB,EAAE,GAAqB,IAAI,CAAC;IAC5B,YAAY,GAAG,KAAK,CAAC;IAE7B,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,GAAG,GAAG,GAAG,QAAQ,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAEpD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,aAAa,GAAG,GAAG,EAAE;gBACzB,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,YAAY,GAAG,CAAC,GAAU,EAAE,EAAE;gBAClC,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC;YAEF,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YAEb,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACf,EAAE,CAAC,kBAAkB,EAAE,CAAC;gBACxB,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,CAAC,IAAI,KAAK,CAAC,2BAA2B,kBAAkB,IAAI,CAAC,CAAC,CAAC;YAC7E,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAEvB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAuB,EAAE,EAAE;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBAC7B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC,CAAC,IAAI,YAAY,WAAW;wBAC3B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;wBACnC,CAAC,CAAC,IAAI,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAC3B,IAAI,GAAkB,CAAC;gBACvB,IAAI,CAAC;oBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;gBACzC,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;oBACnE,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACvB,aAAa,EAAE,CAAC;oBAChB,OAAO;gBACT,CAAC;gBAED,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACd,MAAM,OAAO,GACX,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,uBAAuB,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,SAAS,GAAG,CAAC;oBAC7E,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBACxB,YAAY,CAAC,GAAG,CAAC,CAAC;oBAClB,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC5B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE;gBAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,mBAAmB,CAAC;gBAC3D,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC3C,YAAY,CAAC,IAAI,KAAK,CAAC,yCAAyC,IAAI,MAAM,SAAS,EAAE,CAAC,CAAC,CAAC;YAC1F,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,IAAI,CAAC;YACR,aAAa,EAAE;gBACb,WAAW,EAAE;oBACX;wBACE,QAAQ,EAAE,sBAAsB;wBAChC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;qBAC/B;iBACF;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI,CAAC;YACR,aAAa,EAAE;gBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC5C,YAAY,EAAE,IAAI;aACnB;SACF,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,SAAS;QACP,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK;QACH,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,EAAE,CAAC,kBAAkB,EAAE,CAAC;QACxB,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAEO,SAAS;QACf,MAAM,KAAK,GAA4B;YACrC,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,EAAE;YAC7B,gBAAgB,EAAE;gBAChB,kBAAkB,EAAE,CAAC,OAAO,CAAC;gBAC7B,YAAY,EAAE;oBACZ,WAAW,EAAE;wBACX,mBAAmB,EAAE;4BACnB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM;yBACvC;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,KAAK,CAAC,iBAAiB,GAAG;gBACxB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,KAAK,GAAG;gBACZ;oBACE,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClD,IAAI,EAAE,CAAC,CAAC,IAAI;wBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;wBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;qBACzB,CAAC,CAAC;iBACJ;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACvB,CAAC;IAEO,aAAa,CAAC,GAAkB;QACtC,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,uBAAuB,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,SAAS,GAAG,CAAC;YAC3F,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACvC,OAAO;QACT,CAAC;QAED,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACtB,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,OAA4B;QACtD,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC;YACnD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;gBAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAClE,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAA2B;QACvD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO;oBACL,EAAE;oBACF,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE;iBAC9E,CAAC;YACJ,CAAC;YAED,IAAI,MAAc,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvF,CAAC;YAED,OAAO;gBACL,EAAE;gBACF,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,EAAE,MAAM,EAAE;aACrB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC;YACR,YAAY,EAAE,EAAE,iBAAiB,EAAE,SAAS,EAAE;SAC/C,CAAC,CAAC;IACL,CAAC;IAEO,IAAI,CAAC,IAA6B;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,EAAE,EAAE,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -2,10 +2,12 @@ import { EventEmitter } from 'node:events';
|
|
|
2
2
|
import type { RealtimeSessionConfig } from '../types.js';
|
|
3
3
|
interface OpenAIRealtimeEvents {
|
|
4
4
|
connected: [];
|
|
5
|
+
disconnected: [code: number, reason: string];
|
|
5
6
|
audio: [chunk: Buffer];
|
|
6
7
|
transcript: [text: string, role: 'user' | 'assistant'];
|
|
7
8
|
tool_call: [name: string, args: unknown];
|
|
8
9
|
speech_start: [];
|
|
10
|
+
turn_end: [];
|
|
9
11
|
error: [error: Error];
|
|
10
12
|
}
|
|
11
13
|
export declare class OpenAIRealtimeAdapter extends EventEmitter<OpenAIRealtimeEvents> {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"openai-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAMzD,UAAU,oBAAoB;IAC5B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,qBAAa,qBAAsB,SAAQ,YAAY,CAAC,oBAAoB,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,EAAE,CAA0B;gBAExB,MAAM,EAAE,qBAAqB;IAMnC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAkE9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAO9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAY5B,SAAS,IAAI,IAAI;IAIjB,KAAK,IAAI,IAAI;IAQb,OAAO,CAAC,iBAAiB;IAyBzB,OAAO,CAAC,aAAa;YAyDP,cAAc;IAmC5B,OAAO,CAAC,IAAI;CAMb"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import { WebSocket } from 'ws';
|
|
3
|
-
const DEFAULT_MODEL = 'gpt-4o-mini-realtime
|
|
3
|
+
const DEFAULT_MODEL = 'gpt-4o-mini-realtime';
|
|
4
4
|
const BASE_URL = 'wss://api.openai.com/v1/realtime';
|
|
5
|
+
const CONNECT_TIMEOUT_MS = 30_000;
|
|
5
6
|
export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
6
7
|
config;
|
|
7
8
|
model;
|
|
@@ -12,25 +13,61 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
12
13
|
this.model = config.model ?? DEFAULT_MODEL;
|
|
13
14
|
}
|
|
14
15
|
async connect() {
|
|
16
|
+
if (this.ws) {
|
|
17
|
+
throw new Error('Already connected or connecting — call close() first');
|
|
18
|
+
}
|
|
15
19
|
const url = `${BASE_URL}?model=${this.model}`;
|
|
16
20
|
return new Promise((resolve, reject) => {
|
|
17
|
-
|
|
21
|
+
let settled = false;
|
|
22
|
+
const settleResolve = () => {
|
|
23
|
+
if (settled)
|
|
24
|
+
return;
|
|
25
|
+
settled = true;
|
|
26
|
+
clearTimeout(timer);
|
|
27
|
+
resolve();
|
|
28
|
+
};
|
|
29
|
+
const settleReject = (err) => {
|
|
30
|
+
if (settled)
|
|
31
|
+
return;
|
|
32
|
+
settled = true;
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
reject(err);
|
|
35
|
+
};
|
|
36
|
+
const ws = new WebSocket(url, {
|
|
18
37
|
headers: {
|
|
19
38
|
Authorization: `Bearer ${this.config.apiKey}`,
|
|
20
39
|
'OpenAI-Beta': 'realtime=v1',
|
|
21
40
|
},
|
|
22
41
|
});
|
|
23
|
-
this.ws
|
|
42
|
+
this.ws = ws;
|
|
43
|
+
const timer = setTimeout(() => {
|
|
44
|
+
this.ws = null;
|
|
45
|
+
ws.removeAllListeners();
|
|
46
|
+
ws.close();
|
|
47
|
+
settleReject(new Error(`Connect timed out after ${CONNECT_TIMEOUT_MS}ms`));
|
|
48
|
+
}, CONNECT_TIMEOUT_MS);
|
|
49
|
+
ws.on('open', () => {
|
|
24
50
|
this.sendSessionUpdate();
|
|
25
51
|
this.emit('connected');
|
|
26
|
-
|
|
52
|
+
settleResolve();
|
|
27
53
|
});
|
|
28
|
-
|
|
29
|
-
|
|
54
|
+
ws.on('message', (data) => {
|
|
55
|
+
const buf = Array.isArray(data)
|
|
56
|
+
? Buffer.concat(data)
|
|
57
|
+
: data instanceof ArrayBuffer
|
|
58
|
+
? Buffer.from(new Uint8Array(data))
|
|
59
|
+
: data;
|
|
60
|
+
this.handleMessage(buf.toString());
|
|
30
61
|
});
|
|
31
|
-
|
|
62
|
+
ws.on('error', (err) => {
|
|
32
63
|
this.emit('error', err);
|
|
33
|
-
|
|
64
|
+
settleReject(err);
|
|
65
|
+
});
|
|
66
|
+
ws.on('close', (code, reason) => {
|
|
67
|
+
const reasonStr = reason.toString() || 'connection closed';
|
|
68
|
+
this.ws = null;
|
|
69
|
+
this.emit('disconnected', code, reasonStr);
|
|
70
|
+
settleReject(new Error(`WebSocket closed before connect (code ${code}): ${reasonStr}`));
|
|
34
71
|
});
|
|
35
72
|
});
|
|
36
73
|
}
|
|
@@ -55,14 +92,19 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
55
92
|
this.send({ type: 'response.cancel' });
|
|
56
93
|
}
|
|
57
94
|
close() {
|
|
58
|
-
this.ws
|
|
95
|
+
const ws = this.ws;
|
|
96
|
+
if (!ws)
|
|
97
|
+
return;
|
|
59
98
|
this.ws = null;
|
|
99
|
+
ws.removeAllListeners();
|
|
100
|
+
ws.close();
|
|
60
101
|
}
|
|
61
102
|
sendSessionUpdate() {
|
|
62
103
|
const session = {
|
|
63
104
|
voice: this.config.voice ?? 'coral',
|
|
64
105
|
input_audio_format: 'pcm16',
|
|
65
106
|
output_audio_format: 'pcm16',
|
|
107
|
+
input_audio_transcription: { model: 'whisper-1' },
|
|
66
108
|
turn_detection: { type: 'server_vad' },
|
|
67
109
|
};
|
|
68
110
|
if (this.config.instructions) {
|
|
@@ -89,14 +131,24 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
89
131
|
}
|
|
90
132
|
const type = event.type;
|
|
91
133
|
switch (type) {
|
|
92
|
-
case 'response.audio.delta':
|
|
93
|
-
|
|
134
|
+
case 'response.audio.delta': {
|
|
135
|
+
const delta = event.delta;
|
|
136
|
+
if (typeof delta !== 'string') {
|
|
137
|
+
this.emit('error', new Error('Malformed response.audio.delta: missing or invalid delta'));
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
this.emit('audio', Buffer.from(delta, 'base64'));
|
|
94
141
|
break;
|
|
142
|
+
}
|
|
95
143
|
case 'conversation.item.input_audio_transcription.completed':
|
|
96
|
-
|
|
144
|
+
if (typeof event.transcript === 'string') {
|
|
145
|
+
this.emit('transcript', event.transcript, 'user');
|
|
146
|
+
}
|
|
97
147
|
break;
|
|
98
148
|
case 'response.audio_transcript.done':
|
|
99
|
-
|
|
149
|
+
if (typeof event.transcript === 'string') {
|
|
150
|
+
this.emit('transcript', event.transcript, 'assistant');
|
|
151
|
+
}
|
|
100
152
|
break;
|
|
101
153
|
case 'input_audio_buffer.speech_started':
|
|
102
154
|
this.emit('speech_start');
|
|
@@ -104,9 +156,15 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
104
156
|
case 'response.function_call_arguments.done':
|
|
105
157
|
void this.handleToolCall(event);
|
|
106
158
|
break;
|
|
159
|
+
case 'response.done':
|
|
160
|
+
this.emit('turn_end');
|
|
161
|
+
break;
|
|
107
162
|
case 'error': {
|
|
108
163
|
const errObj = event.error;
|
|
109
|
-
|
|
164
|
+
const message = errObj && typeof errObj === 'object' && 'message' in errObj
|
|
165
|
+
? String(errObj.message)
|
|
166
|
+
: 'Unknown server error';
|
|
167
|
+
this.emit('error', new Error(message));
|
|
110
168
|
break;
|
|
111
169
|
}
|
|
112
170
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-realtime.js","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,
|
|
1
|
+
{"version":3,"file":"openai-realtime.js","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,sBAAsB,CAAC;AAC7C,MAAM,QAAQ,GAAG,kCAAkC,CAAC;AACpD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAalC,MAAM,OAAO,qBAAsB,SAAQ,YAAkC;IAC1D,MAAM,CAAwB;IAC9B,KAAK,CAAS;IACvB,EAAE,GAAqB,IAAI,CAAC;IAEpC,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,GAAG,GAAG,GAAG,QAAQ,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;QAE9C,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,aAAa,GAAG,GAAG,EAAE;gBACzB,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,YAAY,GAAG,CAAC,GAAU,EAAE,EAAE;gBAClC,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC;YAEF,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;oBAC7C,aAAa,EAAE,aAAa;iBAC7B;aACF,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YAEb,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACf,EAAE,CAAC,kBAAkB,EAAE,CAAC;gBACxB,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,CAAC,IAAI,KAAK,CAAC,2BAA2B,kBAAkB,IAAI,CAAC,CAAC,CAAC;YAC7E,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAEvB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACvB,aAAa,EAAE,CAAC;YAClB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAuB,EAAE,EAAE;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBAC7B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC,CAAC,IAAI,YAAY,WAAW;wBAC3B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;wBACnC,CAAC,CAAC,IAAI,CAAC;gBACX,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC5B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxB,YAAY,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE;gBAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,mBAAmB,CAAC;gBAC3D,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC3C,YAAY,CAAC,IAAI,KAAK,CAAC,yCAAyC,IAAI,MAAM,SAAS,EAAE,CAAC,CAAC,CAAC;YAC1F,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACJ,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aACxC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK;QACH,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,EAAE,CAAC,kBAAkB,EAAE,CAAC;QACxB,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAEO,iBAAiB;QACvB,MAAM,OAAO,GAA4B;YACvC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO;YACnC,kBAAkB,EAAE,OAAO;YAC3B,mBAAmB,EAAE,OAAO;YAC5B,yBAAyB,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE;YACjD,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;SACvC,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;aACzB,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,IAAI,KAA8B,CAAC;QACnC,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAc,CAAC;QAElC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,sBAAsB,CAAC,CAAC,CAAC;gBAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC,CAAC;oBAC1F,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACjD,MAAM;YACR,CAAC;YAED,KAAK,uDAAuD;gBAC1D,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;oBACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM;YAER,KAAK,gCAAgC;gBACnC,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;oBACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACzD,CAAC;gBACD,MAAM;YAER,KAAK,mCAAmC;gBACtC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC1B,MAAM;YAER,KAAK,uCAAuC;gBAC1C,KAAK,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM;YAER,KAAK,eAAe;gBAClB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACtB,MAAM;YAER,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC3B,MAAM,OAAO,GACX,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,SAAS,IAAI,MAAM;oBACzD,CAAC,CAAC,MAAM,CAAE,MAAkC,CAAC,OAAO,CAAC;oBACrD,CAAC,CAAC,sBAAsB,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACvC,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAA8B;QACzD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAc,CAAC;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAiB,CAAC;QACvC,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAmB,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,2CAA2C,IAAI,EAAE,CAAC,CAAC,CAAC;YACjF,OAAO;QACT,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAEnC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,MAAM;gBACf,MAAM;aACP;SACF,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACzC,CAAC;IAEO,IAAI,CAAC,IAA6B;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,EAAE,EAAE,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -2,15 +2,19 @@ import { EventEmitter } from 'node:events';
|
|
|
2
2
|
import type { RealtimeSessionConfig } from '../types.js';
|
|
3
3
|
interface RealtimeSessionEvents {
|
|
4
4
|
connected: [];
|
|
5
|
+
disconnected: [code: number, reason: string];
|
|
5
6
|
audio: [chunk: Buffer];
|
|
6
7
|
transcript: [text: string, role: 'user' | 'assistant'];
|
|
7
8
|
tool_call: [name: string, args: unknown];
|
|
8
9
|
speech_start: [];
|
|
10
|
+
turn_end: [];
|
|
9
11
|
error: [error: Error];
|
|
10
12
|
}
|
|
11
13
|
export declare class RealtimeSession extends EventEmitter<RealtimeSessionEvents> {
|
|
12
14
|
private readonly adapter;
|
|
13
15
|
private readonly _provider;
|
|
16
|
+
private readonly forwarders;
|
|
17
|
+
private _closed;
|
|
14
18
|
constructor(config: RealtimeSessionConfig);
|
|
15
19
|
get provider(): RealtimeSessionConfig['provider'];
|
|
16
20
|
connect(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"realtime-session.d.ts","sourceRoot":"","sources":["../../src/realtime/realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"realtime-session.d.ts","sourceRoot":"","sources":["../../src/realtime/realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAiBzD,UAAU,qBAAqB;IAC7B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,qBAAa,eAAgB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IACtE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmD;IAC9E,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,qBAAqB;IAuBzC,IAAI,QAAQ,IAAI,qBAAqB,CAAC,UAAU,CAAC,CAEhD;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5B,SAAS,IAAI,IAAI;IAIjB,KAAK,IAAI,IAAI;CASd"}
|