@cogitator-ai/voice 0.1.13 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -77
- package/dist/audio.d.ts +7 -0
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +40 -3
- package/dist/audio.js.map +1 -1
- package/dist/cogitator-runner.d.ts +34 -0
- package/dist/cogitator-runner.d.ts.map +1 -0
- package/dist/cogitator-runner.js +28 -0
- package/dist/cogitator-runner.js.map +1 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -4
- package/dist/index.js.map +1 -1
- package/dist/pipeline/index.d.ts +2 -2
- package/dist/pipeline/index.d.ts.map +1 -1
- package/dist/pipeline/index.js.map +1 -1
- package/dist/pipeline/pipeline-session.d.ts +20 -2
- package/dist/pipeline/pipeline-session.d.ts.map +1 -1
- package/dist/pipeline/pipeline-session.js +187 -63
- package/dist/pipeline/pipeline-session.js.map +1 -1
- package/dist/pipeline/voice-pipeline.d.ts +12 -7
- package/dist/pipeline/voice-pipeline.d.ts.map +1 -1
- package/dist/pipeline/voice-pipeline.js +14 -4
- package/dist/pipeline/voice-pipeline.js.map +1 -1
- package/dist/realtime/gemini-realtime.d.ts +15 -3
- package/dist/realtime/gemini-realtime.d.ts.map +1 -1
- package/dist/realtime/gemini-realtime.js +140 -94
- package/dist/realtime/gemini-realtime.js.map +1 -1
- package/dist/realtime/openai-realtime.d.ts +10 -1
- package/dist/realtime/openai-realtime.d.ts.map +1 -1
- package/dist/realtime/openai-realtime.js +140 -69
- package/dist/realtime/openai-realtime.js.map +1 -1
- package/dist/realtime/realtime-session.d.ts +10 -0
- package/dist/realtime/realtime-session.d.ts.map +1 -1
- package/dist/realtime/realtime-session.js +60 -7
- package/dist/realtime/realtime-session.js.map +1 -1
- package/dist/stt/deepgram-stt.d.ts +3 -0
- package/dist/stt/deepgram-stt.d.ts.map +1 -1
- package/dist/stt/deepgram-stt.js +104 -55
- package/dist/stt/deepgram-stt.js.map +1 -1
- package/dist/stt/openai-stt.d.ts +5 -0
- package/dist/stt/openai-stt.d.ts.map +1 -1
- package/dist/stt/openai-stt.js +70 -13
- package/dist/stt/openai-stt.js.map +1 -1
- package/dist/tools.d.ts +3 -3
- package/dist/transport/ws-transport.d.ts +6 -7
- package/dist/transport/ws-transport.d.ts.map +1 -1
- package/dist/transport/ws-transport.js +58 -20
- package/dist/transport/ws-transport.js.map +1 -1
- package/dist/tts/elevenlabs-tts.d.ts.map +1 -1
- package/dist/tts/elevenlabs-tts.js +26 -16
- package/dist/tts/elevenlabs-tts.js.map +1 -1
- package/dist/types.d.ts +38 -16
- package/dist/types.d.ts.map +1 -1
- package/dist/vad/silero-vad.d.ts +14 -1
- package/dist/vad/silero-vad.d.ts.map +1 -1
- package/dist/vad/silero-vad.js +101 -25
- package/dist/vad/silero-vad.js.map +1 -1
- package/dist/voice-agent.d.ts +14 -0
- package/dist/voice-agent.d.ts.map +1 -1
- package/dist/voice-agent.js +112 -23
- package/dist/voice-agent.js.map +1 -1
- package/package.json +5 -5
package/dist/pipeline/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { VoicePipeline } from './voice-pipeline.js';
|
|
2
|
-
export { PipelineSession } from './pipeline-session.js';
|
|
1
|
+
export { VoicePipeline, type VoicePipelineResult } from './voice-pipeline.js';
|
|
2
|
+
export { PipelineSession, type PipelineSessionOptions } from './pipeline-session.js';
|
|
3
3
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA4B,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,33 +1,51 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import type { VoicePipelineConfig } from '../types.js';
|
|
3
|
+
type SessionState = 'idle' | 'listening' | 'processing' | 'speaking';
|
|
3
4
|
interface PipelineSessionEvents {
|
|
4
5
|
speech_start: [];
|
|
5
6
|
speech_end: [];
|
|
6
7
|
transcript: [text: string, isFinal: boolean];
|
|
7
8
|
agent_response: [content: string];
|
|
8
9
|
audio: [chunk: Buffer];
|
|
10
|
+
turn_end: [];
|
|
9
11
|
error: [error: Error];
|
|
10
12
|
}
|
|
13
|
+
export interface PipelineSessionOptions {
|
|
14
|
+
sessionId?: string;
|
|
15
|
+
}
|
|
11
16
|
export declare class PipelineSession extends EventEmitter<PipelineSessionEvents> {
|
|
17
|
+
readonly id: string;
|
|
12
18
|
private readonly stt;
|
|
13
19
|
private readonly tts;
|
|
14
20
|
private readonly vad;
|
|
15
21
|
private readonly agent;
|
|
16
22
|
private state;
|
|
17
23
|
private stream;
|
|
18
|
-
private
|
|
24
|
+
private turn;
|
|
19
25
|
private closed;
|
|
20
26
|
private activeProcessing;
|
|
21
27
|
private vadQueue;
|
|
22
|
-
constructor(config: VoicePipelineConfig);
|
|
28
|
+
constructor(config: VoicePipelineConfig, options?: PipelineSessionOptions);
|
|
29
|
+
get currentState(): SessionState;
|
|
23
30
|
pushAudio(chunk: Buffer): void;
|
|
24
31
|
endAudio(): void;
|
|
32
|
+
/**
|
|
33
|
+
* Run an agent turn from text input, bypassing STT. Interrupts any turn in progress.
|
|
34
|
+
*/
|
|
35
|
+
sendText(text: string): Promise<void>;
|
|
25
36
|
interrupt(): void;
|
|
26
37
|
close(): Promise<void>;
|
|
38
|
+
private discardStream;
|
|
27
39
|
private applyVADEvent;
|
|
28
40
|
private handleNoVAD;
|
|
41
|
+
private writeToStream;
|
|
29
42
|
private openStream;
|
|
30
43
|
private finishListening;
|
|
44
|
+
private respond;
|
|
45
|
+
private speak;
|
|
46
|
+
private track;
|
|
47
|
+
private isCurrent;
|
|
48
|
+
private emitError;
|
|
31
49
|
}
|
|
32
50
|
export {};
|
|
33
51
|
//# sourceMappingURL=pipeline-session.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-session.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline-session.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAuB,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAG5E,KAAK,YAAY,GAAG,MAAM,GAAG,WAAW,GAAG,YAAY,GAAG,UAAU,CAAC;AAErE,UAAU,qBAAqB;IAC7B,YAAY,EAAE,EAAE,CAAC;IACjB,UAAU,EAAE,EAAE,CAAC;IACf,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7C,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA4BD,qBAAa,eAAgB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IACtE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,IAAI,CAAyB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,QAAQ,CAAoC;gBAExC,MAAM,EAAE,mBAAmB,EAAE,OAAO,GAAE,sBAA2B;IAS7E,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IA0B9B,QAAQ,IAAI,IAAI;IAShB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQrC,SAAS,IAAI,IAAI;IAQX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,aAAa;IA8BrB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,UAAU;IAiBlB,OAAO,CAAC,eAAe;YAgCT,OAAO;YA8BP,KAAK;IAgBnB,OAAO,CAAC,KAAK;IAWb,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,SAAS;CAIlB"}
|
|
@@ -1,87 +1,147 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { nanoid } from 'nanoid';
|
|
2
3
|
import { pcm16ToFloat32 } from '../audio.js';
|
|
4
|
+
const ABORTED = Symbol('aborted');
|
|
5
|
+
function abortable(promise, signal) {
|
|
6
|
+
if (signal.aborted) {
|
|
7
|
+
promise.catch(() => { });
|
|
8
|
+
return Promise.resolve(ABORTED);
|
|
9
|
+
}
|
|
10
|
+
return new Promise((resolve, reject) => {
|
|
11
|
+
const onAbort = () => {
|
|
12
|
+
promise.catch(() => { });
|
|
13
|
+
resolve(ABORTED);
|
|
14
|
+
};
|
|
15
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
16
|
+
promise.then((value) => {
|
|
17
|
+
signal.removeEventListener('abort', onAbort);
|
|
18
|
+
resolve(value);
|
|
19
|
+
}, (err) => {
|
|
20
|
+
signal.removeEventListener('abort', onAbort);
|
|
21
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
}
|
|
3
25
|
export class PipelineSession extends EventEmitter {
|
|
26
|
+
id;
|
|
4
27
|
stt;
|
|
5
28
|
tts;
|
|
6
29
|
vad;
|
|
7
30
|
agent;
|
|
8
31
|
state = 'idle';
|
|
9
32
|
stream = null;
|
|
10
|
-
|
|
33
|
+
turn = new AbortController();
|
|
11
34
|
closed = false;
|
|
12
35
|
activeProcessing = null;
|
|
13
36
|
vadQueue = Promise.resolve();
|
|
14
|
-
constructor(config) {
|
|
37
|
+
constructor(config, options = {}) {
|
|
15
38
|
super();
|
|
39
|
+
this.id = options.sessionId ?? nanoid();
|
|
16
40
|
this.stt = config.stt;
|
|
17
41
|
this.tts = config.tts;
|
|
18
42
|
this.vad = config.vad;
|
|
19
43
|
this.agent = config.agent;
|
|
20
44
|
}
|
|
45
|
+
get currentState() {
|
|
46
|
+
return this.state;
|
|
47
|
+
}
|
|
21
48
|
pushAudio(chunk) {
|
|
22
49
|
if (this.closed)
|
|
23
50
|
return;
|
|
24
|
-
if (this.vad) {
|
|
25
|
-
const samples = pcm16ToFloat32(chunk);
|
|
26
|
-
const eventOrPromise = this.vad.process(samples);
|
|
27
|
-
this.vadQueue = this.vadQueue.then(async () => {
|
|
28
|
-
const event = await eventOrPromise;
|
|
29
|
-
this.applyVADEvent(event, chunk);
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
else {
|
|
51
|
+
if (!this.vad) {
|
|
33
52
|
this.handleNoVAD(chunk);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
let eventOrPromise;
|
|
56
|
+
try {
|
|
57
|
+
eventOrPromise = this.vad.process(pcm16ToFloat32(chunk));
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
this.emitError(err);
|
|
61
|
+
return;
|
|
34
62
|
}
|
|
63
|
+
this.vadQueue = this.vadQueue.then(async () => {
|
|
64
|
+
try {
|
|
65
|
+
const event = await eventOrPromise;
|
|
66
|
+
if (!this.closed)
|
|
67
|
+
this.applyVADEvent(event, chunk);
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
this.emitError(err);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
35
73
|
}
|
|
36
74
|
endAudio() {
|
|
37
|
-
if (this.
|
|
38
|
-
|
|
39
|
-
|
|
75
|
+
if (this.closed)
|
|
76
|
+
return;
|
|
77
|
+
void this.vadQueue.then(() => {
|
|
78
|
+
if (this.state === 'listening') {
|
|
79
|
+
void this.finishListening();
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Run an agent turn from text input, bypassing STT. Interrupts any turn in progress.
|
|
85
|
+
*/
|
|
86
|
+
sendText(text) {
|
|
87
|
+
if (this.closed)
|
|
88
|
+
return Promise.resolve();
|
|
89
|
+
this.interrupt();
|
|
90
|
+
const signal = this.turn.signal;
|
|
91
|
+
this.state = 'processing';
|
|
92
|
+
return this.track(this.respond(text, signal));
|
|
40
93
|
}
|
|
41
94
|
interrupt() {
|
|
42
|
-
this.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
void this.stream.close().catch(() => { });
|
|
46
|
-
this.stream = null;
|
|
47
|
-
}
|
|
95
|
+
this.turn.abort();
|
|
96
|
+
this.turn = new AbortController();
|
|
97
|
+
this.discardStream();
|
|
48
98
|
this.vad?.reset();
|
|
49
99
|
this.state = 'idle';
|
|
50
100
|
}
|
|
51
101
|
async close() {
|
|
102
|
+
if (this.closed)
|
|
103
|
+
return;
|
|
52
104
|
this.closed = true;
|
|
53
|
-
this.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
void this.stream.close().catch(() => { });
|
|
57
|
-
this.stream = null;
|
|
58
|
-
}
|
|
105
|
+
this.turn.abort();
|
|
106
|
+
this.discardStream();
|
|
107
|
+
this.state = 'idle';
|
|
59
108
|
if (this.activeProcessing) {
|
|
60
109
|
await this.activeProcessing.catch(() => { });
|
|
61
110
|
}
|
|
62
111
|
this.removeAllListeners();
|
|
63
112
|
}
|
|
113
|
+
discardStream() {
|
|
114
|
+
if (!this.stream)
|
|
115
|
+
return;
|
|
116
|
+
const stream = this.stream;
|
|
117
|
+
this.stream = null;
|
|
118
|
+
stream.removeAllListeners();
|
|
119
|
+
void stream.close().catch(() => { });
|
|
120
|
+
}
|
|
64
121
|
applyVADEvent(event, chunk) {
|
|
65
122
|
switch (event.type) {
|
|
66
123
|
case 'speech_start':
|
|
67
124
|
if (this.state === 'idle') {
|
|
68
125
|
this.state = 'listening';
|
|
69
126
|
this.openStream();
|
|
70
|
-
this.
|
|
127
|
+
this.writeToStream(chunk);
|
|
71
128
|
this.emit('speech_start');
|
|
72
129
|
}
|
|
73
130
|
break;
|
|
74
131
|
case 'speech_end':
|
|
75
132
|
if (this.state === 'listening') {
|
|
133
|
+
this.writeToStream(chunk);
|
|
76
134
|
this.emit('speech_end');
|
|
77
135
|
void this.finishListening();
|
|
78
136
|
}
|
|
79
137
|
break;
|
|
80
138
|
case 'speech':
|
|
81
|
-
if (this.state === 'listening'
|
|
82
|
-
this.
|
|
139
|
+
if (this.state === 'listening') {
|
|
140
|
+
this.writeToStream(chunk);
|
|
83
141
|
}
|
|
84
142
|
break;
|
|
143
|
+
case 'silence':
|
|
144
|
+
break;
|
|
85
145
|
}
|
|
86
146
|
}
|
|
87
147
|
handleNoVAD(chunk) {
|
|
@@ -89,65 +149,129 @@ export class PipelineSession extends EventEmitter {
|
|
|
89
149
|
this.state = 'listening';
|
|
90
150
|
this.openStream();
|
|
91
151
|
}
|
|
92
|
-
if (this.state === 'listening'
|
|
152
|
+
if (this.state === 'listening') {
|
|
153
|
+
this.writeToStream(chunk);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
writeToStream(chunk) {
|
|
157
|
+
if (!this.stream)
|
|
158
|
+
return;
|
|
159
|
+
try {
|
|
93
160
|
this.stream.write(chunk);
|
|
94
161
|
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
this.emitError(err);
|
|
164
|
+
}
|
|
95
165
|
}
|
|
96
166
|
openStream() {
|
|
97
|
-
|
|
98
|
-
this.stream
|
|
167
|
+
const stream = this.stt.createStream();
|
|
168
|
+
this.stream = stream;
|
|
169
|
+
stream.on('partial', (text) => {
|
|
99
170
|
this.emit('transcript', text, false);
|
|
100
171
|
});
|
|
101
|
-
|
|
172
|
+
stream.on('final', (result) => {
|
|
102
173
|
this.emit('transcript', result.text, true);
|
|
103
174
|
});
|
|
104
|
-
|
|
105
|
-
this.
|
|
175
|
+
stream.on('error', (error) => {
|
|
176
|
+
this.emitError(error);
|
|
106
177
|
});
|
|
107
178
|
}
|
|
108
179
|
finishListening() {
|
|
109
|
-
if (!this.stream)
|
|
180
|
+
if (!this.stream) {
|
|
181
|
+
this.state = 'idle';
|
|
110
182
|
return Promise.resolve();
|
|
183
|
+
}
|
|
111
184
|
const currentStream = this.stream;
|
|
112
185
|
this.stream = null;
|
|
113
186
|
this.state = 'processing';
|
|
114
|
-
const
|
|
187
|
+
const signal = this.turn.signal;
|
|
188
|
+
return this.track((async () => {
|
|
189
|
+
let transcript;
|
|
115
190
|
try {
|
|
116
|
-
const result = await currentStream.close();
|
|
117
|
-
|
|
118
|
-
if (this.interrupted || this.closed) {
|
|
119
|
-
this.interrupted = false;
|
|
120
|
-
this.state = 'idle';
|
|
191
|
+
const result = await abortable(currentStream.close(), signal);
|
|
192
|
+
if (result === ABORTED)
|
|
121
193
|
return;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (this.
|
|
126
|
-
this.interrupted = false;
|
|
194
|
+
transcript = result.text;
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
if (this.isCurrent(signal)) {
|
|
127
198
|
this.state = 'idle';
|
|
128
|
-
|
|
199
|
+
this.emitError(err);
|
|
129
200
|
}
|
|
130
|
-
|
|
131
|
-
const generator = this.tts.streamSynthesize(content);
|
|
132
|
-
for await (const audioChunk of generator) {
|
|
133
|
-
if (this.interrupted || this.closed)
|
|
134
|
-
break;
|
|
135
|
-
this.emit('audio', audioChunk);
|
|
136
|
-
}
|
|
137
|
-
this.interrupted = false;
|
|
138
|
-
this.state = 'idle';
|
|
201
|
+
return;
|
|
139
202
|
}
|
|
140
|
-
|
|
203
|
+
finally {
|
|
204
|
+
currentStream.removeAllListeners();
|
|
205
|
+
}
|
|
206
|
+
await this.respond(transcript, signal);
|
|
207
|
+
})());
|
|
208
|
+
}
|
|
209
|
+
async respond(input, signal) {
|
|
210
|
+
try {
|
|
211
|
+
if (!this.isCurrent(signal))
|
|
212
|
+
return;
|
|
213
|
+
if (input.trim().length === 0) {
|
|
141
214
|
this.state = 'idle';
|
|
142
|
-
|
|
143
|
-
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
|
215
|
+
return;
|
|
144
216
|
}
|
|
145
|
-
|
|
146
|
-
|
|
217
|
+
const reply = await abortable(this.agent.run(input, { sessionId: this.id, signal }), signal);
|
|
218
|
+
if (reply === ABORTED || !this.isCurrent(signal))
|
|
219
|
+
return;
|
|
220
|
+
const { content } = reply;
|
|
221
|
+
this.emit('agent_response', content);
|
|
222
|
+
if (content.trim().length > 0) {
|
|
223
|
+
this.state = 'speaking';
|
|
224
|
+
const completed = await this.speak(content, signal);
|
|
225
|
+
if (!completed)
|
|
226
|
+
return;
|
|
147
227
|
}
|
|
148
|
-
|
|
228
|
+
if (!this.isCurrent(signal))
|
|
229
|
+
return;
|
|
230
|
+
this.state = 'idle';
|
|
231
|
+
this.emit('turn_end');
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
if (!this.isCurrent(signal))
|
|
235
|
+
return;
|
|
236
|
+
this.state = 'idle';
|
|
237
|
+
this.emitError(err);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
async speak(content, signal) {
|
|
241
|
+
const iterator = this.tts.streamSynthesize(content)[Symbol.asyncIterator]();
|
|
242
|
+
try {
|
|
243
|
+
for (;;) {
|
|
244
|
+
const next = await abortable(iterator.next(), signal);
|
|
245
|
+
if (next === ABORTED || !this.isCurrent(signal))
|
|
246
|
+
return false;
|
|
247
|
+
if (next.done)
|
|
248
|
+
return true;
|
|
249
|
+
this.emit('audio', next.value);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
if (signal.aborted) {
|
|
254
|
+
void iterator.return?.(undefined).catch(() => { });
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
track(processing) {
|
|
149
259
|
this.activeProcessing = processing;
|
|
260
|
+
const release = () => {
|
|
261
|
+
if (this.activeProcessing === processing) {
|
|
262
|
+
this.activeProcessing = null;
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
processing.then(release, release);
|
|
150
266
|
return processing;
|
|
151
267
|
}
|
|
268
|
+
isCurrent(signal) {
|
|
269
|
+
return !this.closed && !signal.aborted;
|
|
270
|
+
}
|
|
271
|
+
emitError(err) {
|
|
272
|
+
if (this.closed)
|
|
273
|
+
return;
|
|
274
|
+
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
|
275
|
+
}
|
|
152
276
|
}
|
|
153
277
|
//# sourceMappingURL=pipeline-session.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-session.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline-session.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAkB7C,MAAM,OAAO,GAAkB,MAAM,CAAC,SAAS,CAAC,CAAC;AAEjD,SAAS,SAAS,CAAI,OAAmB,EAAE,MAAmB;IAC5D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACxB,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACxB,OAAO,CAAC,OAAO,CAAC,CAAC;QACnB,CAAC,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CACV,CAAC,KAAK,EAAE,EAAE;YACR,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC,EACD,CAAC,GAAY,EAAE,EAAE;YACf,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9D,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,eAAgB,SAAQ,YAAmC;IAC7D,EAAE,CAAS;IACH,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,KAAK,CAA+B;IAC7C,KAAK,GAAiB,MAAM,CAAC;IAC7B,MAAM,GAAqB,IAAI,CAAC;IAChC,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;IAC7B,MAAM,GAAG,KAAK,CAAC;IACf,gBAAgB,GAAyB,IAAI,CAAC;IAC9C,QAAQ,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,YAAY,MAA2B,EAAE,UAAkC,EAAE;QAC3E,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAExB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,cAA4C,CAAC;QACjD,IAAI,CAAC;YACH,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACpB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC;gBACnC,IAAI,CAAC,IAAI,CAAC,MAAM;oBAAE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3B,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;gBAC/B,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;QAClC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QAEpB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAe,EAAE,KAAa;QAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;oBAC1B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;oBACzB,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM;YAER,KAAK,YAAY;gBACf,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACxB,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9B,CAAC;gBACD,MAAM;YAER,KAAK,QAAQ;gBACX,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM;YAER,KAAK,SAAS;gBACZ,MAAM;QACV,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa;QAC/B,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;YACzB,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,KAAa;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QACzB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,UAAU;QAChB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;YACpC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,MAAwB,EAAE,EAAE;YAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YAClC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAEhC,OAAO,IAAI,CAAC,KAAK,CACf,CAAC,KAAK,IAAI,EAAE;YACV,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC9D,IAAI,MAAM,KAAK,OAAO;oBAAE,OAAO;gBAC/B,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACtB,CAAC;gBACD,OAAO;YACT,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,kBAAkB,EAAE,CAAC;YACrC,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC,CAAC,EAAE,CACL,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,MAAmB;QACtD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YAEpC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;gBACpB,OAAO;YACT,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;YAC7F,IAAI,KAAK,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YACzD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;YAErC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;gBACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBACpD,IAAI,CAAC,SAAS;oBAAE,OAAO;YACzB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YACpC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YACpC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,OAAe,EAAE,MAAmB;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAC5E,IAAI,CAAC;YACH,SAAS,CAAC;gBACR,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;gBACtD,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC9D,IAAI,IAAI,CAAC,IAAI;oBAAE,OAAO,IAAI,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAyB;QACrC,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,IAAI,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;gBACzC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC;QACF,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,SAAS,CAAC,MAAmB;QACnC,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;IACzC,CAAC;IAEO,SAAS,CAAC,GAAY;QAC5B,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF"}
|
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
import type { VoicePipelineConfig } from '../types.js';
|
|
2
|
-
import { PipelineSession } from './pipeline-session.js';
|
|
2
|
+
import { PipelineSession, type PipelineSessionOptions } from './pipeline-session.js';
|
|
3
|
+
export interface VoicePipelineResult {
|
|
4
|
+
transcript: string;
|
|
5
|
+
response: string;
|
|
6
|
+
audio: Buffer;
|
|
7
|
+
}
|
|
3
8
|
export declare class VoicePipeline {
|
|
4
9
|
private readonly config;
|
|
5
10
|
constructor(config: VoicePipelineConfig);
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
createSession(): PipelineSession;
|
|
11
|
+
/**
|
|
12
|
+
* One-shot STT -> Agent -> TTS. Silent input (empty transcript) skips the agent and TTS
|
|
13
|
+
* and returns empty strings with an empty audio buffer.
|
|
14
|
+
*/
|
|
15
|
+
process(audio: Buffer, options?: PipelineSessionOptions): Promise<VoicePipelineResult>;
|
|
16
|
+
createSession(options?: PipelineSessionOptions): PipelineSession;
|
|
12
17
|
}
|
|
13
18
|
//# sourceMappingURL=voice-pipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voice-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"voice-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAErF,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAEjC,MAAM,EAAE,mBAAmB;IAIvC;;;OAGG;IACG,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAkBhG,aAAa,CAAC,OAAO,CAAC,EAAE,sBAAsB,GAAG,eAAe;CAGjE"}
|
|
@@ -4,14 +4,24 @@ export class VoicePipeline {
|
|
|
4
4
|
constructor(config) {
|
|
5
5
|
this.config = config;
|
|
6
6
|
}
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* One-shot STT -> Agent -> TTS. Silent input (empty transcript) skips the agent and TTS
|
|
9
|
+
* and returns empty strings with an empty audio buffer.
|
|
10
|
+
*/
|
|
11
|
+
async process(audio, options = {}) {
|
|
8
12
|
const { text: transcript } = await this.config.stt.transcribe(audio);
|
|
9
|
-
|
|
13
|
+
if (transcript.trim().length === 0) {
|
|
14
|
+
return { transcript, response: '', audio: Buffer.alloc(0) };
|
|
15
|
+
}
|
|
16
|
+
const { content: response } = await this.config.agent.run(transcript, options.sessionId ? { sessionId: options.sessionId } : undefined);
|
|
17
|
+
if (response.trim().length === 0) {
|
|
18
|
+
return { transcript, response, audio: Buffer.alloc(0) };
|
|
19
|
+
}
|
|
10
20
|
const outputAudio = await this.config.tts.synthesize(response);
|
|
11
21
|
return { transcript, response, audio: outputAudio };
|
|
12
22
|
}
|
|
13
|
-
createSession() {
|
|
14
|
-
return new PipelineSession(this.config);
|
|
23
|
+
createSession(options) {
|
|
24
|
+
return new PipelineSession(this.config, options);
|
|
15
25
|
}
|
|
16
26
|
}
|
|
17
27
|
//# sourceMappingURL=voice-pipeline.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voice-pipeline.js","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"voice-pipeline.js","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AAQrF,MAAM,OAAO,aAAa;IACP,MAAM,CAAsB;IAE7C,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,UAAkC,EAAE;QAC/D,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CACvD,UAAU,EACV,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CACjE,CAAC;QACF,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC/D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IACtD,CAAC;IAED,aAAa,CAAC,OAAgC;QAC5C,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;CACF"}
|
|
@@ -10,25 +10,37 @@ interface GeminiRealtimeEvents {
|
|
|
10
10
|
turn_end: [];
|
|
11
11
|
error: [error: Error];
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* Gemini Live API adapter. Input audio is PCM16 mono at 16kHz,
|
|
15
|
+
* output audio is PCM16 mono at 24kHz.
|
|
16
|
+
*/
|
|
13
17
|
export declare class GeminiRealtimeAdapter extends EventEmitter<GeminiRealtimeEvents> {
|
|
14
18
|
private readonly config;
|
|
15
19
|
private readonly model;
|
|
16
20
|
private ws;
|
|
21
|
+
private connected;
|
|
22
|
+
private modelTurnActive;
|
|
17
23
|
private interrupting;
|
|
24
|
+
private userTranscript;
|
|
25
|
+
private assistantTranscript;
|
|
26
|
+
private assistantText;
|
|
18
27
|
constructor(config: RealtimeSessionConfig);
|
|
28
|
+
get isConnected(): boolean;
|
|
19
29
|
connect(): Promise<void>;
|
|
20
30
|
pushAudio(chunk: Buffer): void;
|
|
21
31
|
sendText(text: string): void;
|
|
22
32
|
/**
|
|
23
33
|
* Gemini Live barge-in is driven by incoming user audio, not a control message.
|
|
24
|
-
*
|
|
25
|
-
* turn completes
|
|
34
|
+
* While a model turn is in progress this drops the remaining model audio until
|
|
35
|
+
* the turn completes, simulating an interruption on the consumer side.
|
|
26
36
|
*/
|
|
27
37
|
interrupt(): void;
|
|
28
38
|
close(): void;
|
|
29
39
|
private sendSetup;
|
|
30
|
-
private handleMessage;
|
|
31
40
|
private handleServerContent;
|
|
41
|
+
private flushUserTranscript;
|
|
42
|
+
private flushAssistantTranscript;
|
|
43
|
+
private resetTurn;
|
|
32
44
|
private handleToolCalls;
|
|
33
45
|
private send;
|
|
34
46
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AASzD,UAAU,oBAAoB;IAC5B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAmCD;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,YAAY,CAAC,oBAAoB,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,cAAc,CAAM;IAC5B,OAAO,CAAC,mBAAmB,CAAM;IACjC,OAAO,CAAC,aAAa,CAAM;gBAEf,MAAM,EAAE,qBAAqB;IAMzC,IAAI,WAAW,IAAI,OAAO,CAEzB;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAmF9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAQ9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5B;;;;OAIG;IACH,SAAS,IAAI,IAAI;IAMjB,KAAK,IAAI,IAAI;IAWb,OAAO,CAAC,SAAS;IAkCjB,OAAO,CAAC,mBAAmB;IA2C3B,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,wBAAwB;IAOhC,OAAO,CAAC,SAAS;YAQH,eAAe;IA8B7B,OAAO,CAAC,IAAI;CAMb"}
|