@cogitator-ai/voice 0.1.13 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -84
- package/dist/audio.d.ts +7 -0
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +40 -3
- package/dist/audio.js.map +1 -1
- package/dist/cogitator-runner.d.ts +34 -0
- package/dist/cogitator-runner.d.ts.map +1 -0
- package/dist/cogitator-runner.js +28 -0
- package/dist/cogitator-runner.js.map +1 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -4
- package/dist/index.js.map +1 -1
- package/dist/pipeline/index.d.ts +2 -2
- package/dist/pipeline/index.d.ts.map +1 -1
- package/dist/pipeline/index.js.map +1 -1
- package/dist/pipeline/pipeline-session.d.ts +20 -2
- package/dist/pipeline/pipeline-session.d.ts.map +1 -1
- package/dist/pipeline/pipeline-session.js +187 -63
- package/dist/pipeline/pipeline-session.js.map +1 -1
- package/dist/pipeline/voice-pipeline.d.ts +12 -7
- package/dist/pipeline/voice-pipeline.d.ts.map +1 -1
- package/dist/pipeline/voice-pipeline.js +14 -4
- package/dist/pipeline/voice-pipeline.js.map +1 -1
- package/dist/realtime/gemini-realtime.d.ts +15 -3
- package/dist/realtime/gemini-realtime.d.ts.map +1 -1
- package/dist/realtime/gemini-realtime.js +140 -94
- package/dist/realtime/gemini-realtime.js.map +1 -1
- package/dist/realtime/openai-realtime.d.ts +10 -1
- package/dist/realtime/openai-realtime.d.ts.map +1 -1
- package/dist/realtime/openai-realtime.js +140 -69
- package/dist/realtime/openai-realtime.js.map +1 -1
- package/dist/realtime/realtime-session.d.ts +10 -0
- package/dist/realtime/realtime-session.d.ts.map +1 -1
- package/dist/realtime/realtime-session.js +60 -7
- package/dist/realtime/realtime-session.js.map +1 -1
- package/dist/stt/deepgram-stt.d.ts +3 -0
- package/dist/stt/deepgram-stt.d.ts.map +1 -1
- package/dist/stt/deepgram-stt.js +104 -55
- package/dist/stt/deepgram-stt.js.map +1 -1
- package/dist/stt/openai-stt.d.ts +5 -0
- package/dist/stt/openai-stt.d.ts.map +1 -1
- package/dist/stt/openai-stt.js +87 -17
- package/dist/stt/openai-stt.js.map +1 -1
- package/dist/tools.d.ts +3 -3
- package/dist/transport/ws-transport.d.ts +6 -7
- package/dist/transport/ws-transport.d.ts.map +1 -1
- package/dist/transport/ws-transport.js +58 -20
- package/dist/transport/ws-transport.js.map +1 -1
- package/dist/tts/elevenlabs-tts.d.ts.map +1 -1
- package/dist/tts/elevenlabs-tts.js +26 -16
- package/dist/tts/elevenlabs-tts.js.map +1 -1
- package/dist/types.d.ts +38 -16
- package/dist/types.d.ts.map +1 -1
- package/dist/vad/silero-vad.d.ts +14 -1
- package/dist/vad/silero-vad.d.ts.map +1 -1
- package/dist/vad/silero-vad.js +101 -25
- package/dist/vad/silero-vad.js.map +1 -1
- package/dist/voice-agent.d.ts +14 -0
- package/dist/voice-agent.d.ts.map +1 -1
- package/dist/voice-agent.js +112 -23
- package/dist/voice-agent.js.map +1 -1
- package/package.json +15 -13
package/dist/index.js
CHANGED
|
@@ -1,16 +1,19 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
2
|
+
const packageJson = createRequire(import.meta.url)('../package.json');
|
|
3
|
+
export const VERSION = packageJson.version;
|
|
4
|
+
export { float32ToPcm16, pcm16ToFloat32, pcmToWav, wavToPcm, resample, calculateRMS, detectAudioFormat, audioMimeType, } from './audio.js';
|
|
3
5
|
export { OpenAISTT } from './stt/index.js';
|
|
4
6
|
export { DeepgramSTT } from './stt/index.js';
|
|
5
7
|
export { OpenAITTS } from './tts/index.js';
|
|
6
8
|
export { ElevenLabsTTS } from './tts/index.js';
|
|
7
9
|
export { EnergyVAD } from './vad/index.js';
|
|
8
10
|
export { SileroVAD } from './vad/index.js';
|
|
9
|
-
export { VoicePipeline, PipelineSession } from './pipeline/index.js';
|
|
11
|
+
export { VoicePipeline, PipelineSession, } from './pipeline/index.js';
|
|
10
12
|
export { OpenAIRealtimeAdapter } from './realtime/index.js';
|
|
11
13
|
export { GeminiRealtimeAdapter } from './realtime/index.js';
|
|
12
14
|
export { RealtimeSession } from './realtime/index.js';
|
|
13
|
-
export { WebSocketTransport, VoiceClient } from './transport/index.js';
|
|
15
|
+
export { WebSocketTransport, VoiceClient, } from './transport/index.js';
|
|
14
16
|
export { VoiceAgent } from './voice-agent.js';
|
|
17
|
+
export { createCogitatorRunner, } from './cogitator-runner.js';
|
|
15
18
|
export { transcribeTool, speakTool, voiceTools } from './tools.js';
|
|
16
19
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAuBnD,OAAO,EACL,cAAc,EACd,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,YAAY,EACZ,iBAAiB,EACjB,aAAa,GAEd,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAwB,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AAErE,OAAO,EAAE,SAAS,EAAwB,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,aAAa,EAA4B,MAAM,gBAAgB,CAAC;AAEzE,OAAO,EAAE,SAAS,EAAwB,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,SAAS,EAAwB,MAAM,gBAAgB,CAAC;AAEjE,OAAO,EACL,aAAa,EACb,eAAe,GAGhB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EACL,kBAAkB,EAClB,WAAW,GAEZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EACL,qBAAqB,GAGtB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAkB,MAAM,YAAY,CAAC"}
|
package/dist/pipeline/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { VoicePipeline } from './voice-pipeline.js';
|
|
2
|
-
export { PipelineSession } from './pipeline-session.js';
|
|
1
|
+
export { VoicePipeline, type VoicePipelineResult } from './voice-pipeline.js';
|
|
2
|
+
export { PipelineSession, type PipelineSessionOptions } from './pipeline-session.js';
|
|
3
3
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA4B,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,33 +1,51 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import type { VoicePipelineConfig } from '../types.js';
|
|
3
|
+
type SessionState = 'idle' | 'listening' | 'processing' | 'speaking';
|
|
3
4
|
interface PipelineSessionEvents {
|
|
4
5
|
speech_start: [];
|
|
5
6
|
speech_end: [];
|
|
6
7
|
transcript: [text: string, isFinal: boolean];
|
|
7
8
|
agent_response: [content: string];
|
|
8
9
|
audio: [chunk: Buffer];
|
|
10
|
+
turn_end: [];
|
|
9
11
|
error: [error: Error];
|
|
10
12
|
}
|
|
13
|
+
export interface PipelineSessionOptions {
|
|
14
|
+
sessionId?: string;
|
|
15
|
+
}
|
|
11
16
|
export declare class PipelineSession extends EventEmitter<PipelineSessionEvents> {
|
|
17
|
+
readonly id: string;
|
|
12
18
|
private readonly stt;
|
|
13
19
|
private readonly tts;
|
|
14
20
|
private readonly vad;
|
|
15
21
|
private readonly agent;
|
|
16
22
|
private state;
|
|
17
23
|
private stream;
|
|
18
|
-
private
|
|
24
|
+
private turn;
|
|
19
25
|
private closed;
|
|
20
26
|
private activeProcessing;
|
|
21
27
|
private vadQueue;
|
|
22
|
-
constructor(config: VoicePipelineConfig);
|
|
28
|
+
constructor(config: VoicePipelineConfig, options?: PipelineSessionOptions);
|
|
29
|
+
get currentState(): SessionState;
|
|
23
30
|
pushAudio(chunk: Buffer): void;
|
|
24
31
|
endAudio(): void;
|
|
32
|
+
/**
|
|
33
|
+
* Run an agent turn from text input, bypassing STT. Interrupts any turn in progress.
|
|
34
|
+
*/
|
|
35
|
+
sendText(text: string): Promise<void>;
|
|
25
36
|
interrupt(): void;
|
|
26
37
|
close(): Promise<void>;
|
|
38
|
+
private discardStream;
|
|
27
39
|
private applyVADEvent;
|
|
28
40
|
private handleNoVAD;
|
|
41
|
+
private writeToStream;
|
|
29
42
|
private openStream;
|
|
30
43
|
private finishListening;
|
|
44
|
+
private respond;
|
|
45
|
+
private speak;
|
|
46
|
+
private track;
|
|
47
|
+
private isCurrent;
|
|
48
|
+
private emitError;
|
|
31
49
|
}
|
|
32
50
|
export {};
|
|
33
51
|
//# sourceMappingURL=pipeline-session.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-session.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline-session.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAuB,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAG5E,KAAK,YAAY,GAAG,MAAM,GAAG,WAAW,GAAG,YAAY,GAAG,UAAU,CAAC;AAErE,UAAU,qBAAqB;IAC7B,YAAY,EAAE,EAAE,CAAC;IACjB,UAAU,EAAE,EAAE,CAAC;IACf,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7C,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAClC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AA4BD,qBAAa,eAAgB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IACtE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,IAAI,CAAyB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,QAAQ,CAAoC;gBAExC,MAAM,EAAE,mBAAmB,EAAE,OAAO,GAAE,sBAA2B;IAS7E,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IA0B9B,QAAQ,IAAI,IAAI;IAShB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQrC,SAAS,IAAI,IAAI;IAQX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,aAAa;IA8BrB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,UAAU;IAiBlB,OAAO,CAAC,eAAe;YAgCT,OAAO;YA8BP,KAAK;IAgBnB,OAAO,CAAC,KAAK;IAWb,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,SAAS;CAIlB"}
|
|
@@ -1,87 +1,147 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { nanoid } from 'nanoid';
|
|
2
3
|
import { pcm16ToFloat32 } from '../audio.js';
|
|
4
|
+
const ABORTED = Symbol('aborted');
|
|
5
|
+
function abortable(promise, signal) {
|
|
6
|
+
if (signal.aborted) {
|
|
7
|
+
promise.catch(() => { });
|
|
8
|
+
return Promise.resolve(ABORTED);
|
|
9
|
+
}
|
|
10
|
+
return new Promise((resolve, reject) => {
|
|
11
|
+
const onAbort = () => {
|
|
12
|
+
promise.catch(() => { });
|
|
13
|
+
resolve(ABORTED);
|
|
14
|
+
};
|
|
15
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
16
|
+
promise.then((value) => {
|
|
17
|
+
signal.removeEventListener('abort', onAbort);
|
|
18
|
+
resolve(value);
|
|
19
|
+
}, (err) => {
|
|
20
|
+
signal.removeEventListener('abort', onAbort);
|
|
21
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
}
|
|
3
25
|
export class PipelineSession extends EventEmitter {
|
|
26
|
+
id;
|
|
4
27
|
stt;
|
|
5
28
|
tts;
|
|
6
29
|
vad;
|
|
7
30
|
agent;
|
|
8
31
|
state = 'idle';
|
|
9
32
|
stream = null;
|
|
10
|
-
|
|
33
|
+
turn = new AbortController();
|
|
11
34
|
closed = false;
|
|
12
35
|
activeProcessing = null;
|
|
13
36
|
vadQueue = Promise.resolve();
|
|
14
|
-
constructor(config) {
|
|
37
|
+
constructor(config, options = {}) {
|
|
15
38
|
super();
|
|
39
|
+
this.id = options.sessionId ?? nanoid();
|
|
16
40
|
this.stt = config.stt;
|
|
17
41
|
this.tts = config.tts;
|
|
18
42
|
this.vad = config.vad;
|
|
19
43
|
this.agent = config.agent;
|
|
20
44
|
}
|
|
45
|
+
get currentState() {
|
|
46
|
+
return this.state;
|
|
47
|
+
}
|
|
21
48
|
pushAudio(chunk) {
|
|
22
49
|
if (this.closed)
|
|
23
50
|
return;
|
|
24
|
-
if (this.vad) {
|
|
25
|
-
const samples = pcm16ToFloat32(chunk);
|
|
26
|
-
const eventOrPromise = this.vad.process(samples);
|
|
27
|
-
this.vadQueue = this.vadQueue.then(async () => {
|
|
28
|
-
const event = await eventOrPromise;
|
|
29
|
-
this.applyVADEvent(event, chunk);
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
else {
|
|
51
|
+
if (!this.vad) {
|
|
33
52
|
this.handleNoVAD(chunk);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
let eventOrPromise;
|
|
56
|
+
try {
|
|
57
|
+
eventOrPromise = this.vad.process(pcm16ToFloat32(chunk));
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
this.emitError(err);
|
|
61
|
+
return;
|
|
34
62
|
}
|
|
63
|
+
this.vadQueue = this.vadQueue.then(async () => {
|
|
64
|
+
try {
|
|
65
|
+
const event = await eventOrPromise;
|
|
66
|
+
if (!this.closed)
|
|
67
|
+
this.applyVADEvent(event, chunk);
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
this.emitError(err);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
35
73
|
}
|
|
36
74
|
endAudio() {
|
|
37
|
-
if (this.
|
|
38
|
-
|
|
39
|
-
|
|
75
|
+
if (this.closed)
|
|
76
|
+
return;
|
|
77
|
+
void this.vadQueue.then(() => {
|
|
78
|
+
if (this.state === 'listening') {
|
|
79
|
+
void this.finishListening();
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Run an agent turn from text input, bypassing STT. Interrupts any turn in progress.
|
|
85
|
+
*/
|
|
86
|
+
sendText(text) {
|
|
87
|
+
if (this.closed)
|
|
88
|
+
return Promise.resolve();
|
|
89
|
+
this.interrupt();
|
|
90
|
+
const signal = this.turn.signal;
|
|
91
|
+
this.state = 'processing';
|
|
92
|
+
return this.track(this.respond(text, signal));
|
|
40
93
|
}
|
|
41
94
|
interrupt() {
|
|
42
|
-
this.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
void this.stream.close().catch(() => { });
|
|
46
|
-
this.stream = null;
|
|
47
|
-
}
|
|
95
|
+
this.turn.abort();
|
|
96
|
+
this.turn = new AbortController();
|
|
97
|
+
this.discardStream();
|
|
48
98
|
this.vad?.reset();
|
|
49
99
|
this.state = 'idle';
|
|
50
100
|
}
|
|
51
101
|
async close() {
|
|
102
|
+
if (this.closed)
|
|
103
|
+
return;
|
|
52
104
|
this.closed = true;
|
|
53
|
-
this.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
void this.stream.close().catch(() => { });
|
|
57
|
-
this.stream = null;
|
|
58
|
-
}
|
|
105
|
+
this.turn.abort();
|
|
106
|
+
this.discardStream();
|
|
107
|
+
this.state = 'idle';
|
|
59
108
|
if (this.activeProcessing) {
|
|
60
109
|
await this.activeProcessing.catch(() => { });
|
|
61
110
|
}
|
|
62
111
|
this.removeAllListeners();
|
|
63
112
|
}
|
|
113
|
+
discardStream() {
|
|
114
|
+
if (!this.stream)
|
|
115
|
+
return;
|
|
116
|
+
const stream = this.stream;
|
|
117
|
+
this.stream = null;
|
|
118
|
+
stream.removeAllListeners();
|
|
119
|
+
void stream.close().catch(() => { });
|
|
120
|
+
}
|
|
64
121
|
applyVADEvent(event, chunk) {
|
|
65
122
|
switch (event.type) {
|
|
66
123
|
case 'speech_start':
|
|
67
124
|
if (this.state === 'idle') {
|
|
68
125
|
this.state = 'listening';
|
|
69
126
|
this.openStream();
|
|
70
|
-
this.
|
|
127
|
+
this.writeToStream(chunk);
|
|
71
128
|
this.emit('speech_start');
|
|
72
129
|
}
|
|
73
130
|
break;
|
|
74
131
|
case 'speech_end':
|
|
75
132
|
if (this.state === 'listening') {
|
|
133
|
+
this.writeToStream(chunk);
|
|
76
134
|
this.emit('speech_end');
|
|
77
135
|
void this.finishListening();
|
|
78
136
|
}
|
|
79
137
|
break;
|
|
80
138
|
case 'speech':
|
|
81
|
-
if (this.state === 'listening'
|
|
82
|
-
this.
|
|
139
|
+
if (this.state === 'listening') {
|
|
140
|
+
this.writeToStream(chunk);
|
|
83
141
|
}
|
|
84
142
|
break;
|
|
143
|
+
case 'silence':
|
|
144
|
+
break;
|
|
85
145
|
}
|
|
86
146
|
}
|
|
87
147
|
handleNoVAD(chunk) {
|
|
@@ -89,65 +149,129 @@ export class PipelineSession extends EventEmitter {
|
|
|
89
149
|
this.state = 'listening';
|
|
90
150
|
this.openStream();
|
|
91
151
|
}
|
|
92
|
-
if (this.state === 'listening'
|
|
152
|
+
if (this.state === 'listening') {
|
|
153
|
+
this.writeToStream(chunk);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
writeToStream(chunk) {
|
|
157
|
+
if (!this.stream)
|
|
158
|
+
return;
|
|
159
|
+
try {
|
|
93
160
|
this.stream.write(chunk);
|
|
94
161
|
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
this.emitError(err);
|
|
164
|
+
}
|
|
95
165
|
}
|
|
96
166
|
openStream() {
|
|
97
|
-
|
|
98
|
-
this.stream
|
|
167
|
+
const stream = this.stt.createStream();
|
|
168
|
+
this.stream = stream;
|
|
169
|
+
stream.on('partial', (text) => {
|
|
99
170
|
this.emit('transcript', text, false);
|
|
100
171
|
});
|
|
101
|
-
|
|
172
|
+
stream.on('final', (result) => {
|
|
102
173
|
this.emit('transcript', result.text, true);
|
|
103
174
|
});
|
|
104
|
-
|
|
105
|
-
this.
|
|
175
|
+
stream.on('error', (error) => {
|
|
176
|
+
this.emitError(error);
|
|
106
177
|
});
|
|
107
178
|
}
|
|
108
179
|
finishListening() {
|
|
109
|
-
if (!this.stream)
|
|
180
|
+
if (!this.stream) {
|
|
181
|
+
this.state = 'idle';
|
|
110
182
|
return Promise.resolve();
|
|
183
|
+
}
|
|
111
184
|
const currentStream = this.stream;
|
|
112
185
|
this.stream = null;
|
|
113
186
|
this.state = 'processing';
|
|
114
|
-
const
|
|
187
|
+
const signal = this.turn.signal;
|
|
188
|
+
return this.track((async () => {
|
|
189
|
+
let transcript;
|
|
115
190
|
try {
|
|
116
|
-
const result = await currentStream.close();
|
|
117
|
-
|
|
118
|
-
if (this.interrupted || this.closed) {
|
|
119
|
-
this.interrupted = false;
|
|
120
|
-
this.state = 'idle';
|
|
191
|
+
const result = await abortable(currentStream.close(), signal);
|
|
192
|
+
if (result === ABORTED)
|
|
121
193
|
return;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (this.
|
|
126
|
-
this.interrupted = false;
|
|
194
|
+
transcript = result.text;
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
if (this.isCurrent(signal)) {
|
|
127
198
|
this.state = 'idle';
|
|
128
|
-
|
|
199
|
+
this.emitError(err);
|
|
129
200
|
}
|
|
130
|
-
|
|
131
|
-
const generator = this.tts.streamSynthesize(content);
|
|
132
|
-
for await (const audioChunk of generator) {
|
|
133
|
-
if (this.interrupted || this.closed)
|
|
134
|
-
break;
|
|
135
|
-
this.emit('audio', audioChunk);
|
|
136
|
-
}
|
|
137
|
-
this.interrupted = false;
|
|
138
|
-
this.state = 'idle';
|
|
201
|
+
return;
|
|
139
202
|
}
|
|
140
|
-
|
|
203
|
+
finally {
|
|
204
|
+
currentStream.removeAllListeners();
|
|
205
|
+
}
|
|
206
|
+
await this.respond(transcript, signal);
|
|
207
|
+
})());
|
|
208
|
+
}
|
|
209
|
+
async respond(input, signal) {
|
|
210
|
+
try {
|
|
211
|
+
if (!this.isCurrent(signal))
|
|
212
|
+
return;
|
|
213
|
+
if (input.trim().length === 0) {
|
|
141
214
|
this.state = 'idle';
|
|
142
|
-
|
|
143
|
-
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
|
215
|
+
return;
|
|
144
216
|
}
|
|
145
|
-
|
|
146
|
-
|
|
217
|
+
const reply = await abortable(this.agent.run(input, { sessionId: this.id, signal }), signal);
|
|
218
|
+
if (reply === ABORTED || !this.isCurrent(signal))
|
|
219
|
+
return;
|
|
220
|
+
const { content } = reply;
|
|
221
|
+
this.emit('agent_response', content);
|
|
222
|
+
if (content.trim().length > 0) {
|
|
223
|
+
this.state = 'speaking';
|
|
224
|
+
const completed = await this.speak(content, signal);
|
|
225
|
+
if (!completed)
|
|
226
|
+
return;
|
|
147
227
|
}
|
|
148
|
-
|
|
228
|
+
if (!this.isCurrent(signal))
|
|
229
|
+
return;
|
|
230
|
+
this.state = 'idle';
|
|
231
|
+
this.emit('turn_end');
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
if (!this.isCurrent(signal))
|
|
235
|
+
return;
|
|
236
|
+
this.state = 'idle';
|
|
237
|
+
this.emitError(err);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
async speak(content, signal) {
|
|
241
|
+
const iterator = this.tts.streamSynthesize(content)[Symbol.asyncIterator]();
|
|
242
|
+
try {
|
|
243
|
+
for (;;) {
|
|
244
|
+
const next = await abortable(iterator.next(), signal);
|
|
245
|
+
if (next === ABORTED || !this.isCurrent(signal))
|
|
246
|
+
return false;
|
|
247
|
+
if (next.done)
|
|
248
|
+
return true;
|
|
249
|
+
this.emit('audio', next.value);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
if (signal.aborted) {
|
|
254
|
+
void iterator.return?.(undefined).catch(() => { });
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
track(processing) {
|
|
149
259
|
this.activeProcessing = processing;
|
|
260
|
+
const release = () => {
|
|
261
|
+
if (this.activeProcessing === processing) {
|
|
262
|
+
this.activeProcessing = null;
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
processing.then(release, release);
|
|
150
266
|
return processing;
|
|
151
267
|
}
|
|
268
|
+
isCurrent(signal) {
|
|
269
|
+
return !this.closed && !signal.aborted;
|
|
270
|
+
}
|
|
271
|
+
emitError(err) {
|
|
272
|
+
if (this.closed)
|
|
273
|
+
return;
|
|
274
|
+
this.emit('error', err instanceof Error ? err : new Error(String(err)));
|
|
275
|
+
}
|
|
152
276
|
}
|
|
153
277
|
//# sourceMappingURL=pipeline-session.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-session.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline-session.js","sourceRoot":"","sources":["../../src/pipeline/pipeline-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAkB7C,MAAM,OAAO,GAAkB,MAAM,CAAC,SAAS,CAAC,CAAC;AAEjD,SAAS,SAAS,CAAI,OAAmB,EAAE,MAAmB;IAC5D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACxB,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACxB,OAAO,CAAC,OAAO,CAAC,CAAC;QACnB,CAAC,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CACV,CAAC,KAAK,EAAE,EAAE;YACR,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC,EACD,CAAC,GAAY,EAAE,EAAE;YACf,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9D,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,eAAgB,SAAQ,YAAmC;IAC7D,EAAE,CAAS;IACH,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,GAAG,CAA6B;IAChC,KAAK,CAA+B;IAC7C,KAAK,GAAiB,MAAM,CAAC;IAC7B,MAAM,GAAqB,IAAI,CAAC;IAChC,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;IAC7B,MAAM,GAAG,KAAK,CAAC;IACf,gBAAgB,GAAyB,IAAI,CAAC;IAC9C,QAAQ,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,YAAY,MAA2B,EAAE,UAAkC,EAAE;QAC3E,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAExB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,cAA4C,CAAC;QACjD,IAAI,CAAC;YACH,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACpB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC;gBACnC,IAAI,CAAC,IAAI,CAAC,MAAM;oBAAE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3B,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;gBAC/B,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;QAClC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QAEpB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAe,EAAE,KAAa;QAClD,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;oBAC1B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;oBACzB,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM;YAER,KAAK,YAAY;gBACf,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACxB,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC9B,CAAC;gBACD,MAAM;YAER,KAAK,QAAQ;gBACX,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM;YAER,KAAK,SAAS;gBACZ,MAAM;QACV,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa;QAC/B,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;YACzB,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,KAAa;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QACzB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,UAAU;QAChB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAY,EAAE,EAAE;YACpC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,MAAwB,EAAE,EAAE;YAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YAClC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAEhC,OAAO,IAAI,CAAC,KAAK,CACf,CAAC,KAAK,IAAI,EAAE;YACV,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC9D,IAAI,MAAM,KAAK,OAAO;oBAAE,OAAO;gBAC/B,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;oBACpB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACtB,CAAC;gBACD,OAAO;YACT,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,kBAAkB,EAAE,CAAC;YACrC,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC,CAAC,EAAE,CACL,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,MAAmB;QACtD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YAEpC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;gBACpB,OAAO;YACT,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;YAC7F,IAAI,KAAK,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YACzD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;YAErC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;gBACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBACpD,IAAI,CAAC,SAAS;oBAAE,OAAO;YACzB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YACpC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO;YACpC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,OAAe,EAAE,MAAmB;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAC5E,IAAI,CAAC;YACH,SAAS,CAAC;gBACR,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;gBACtD,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAC9D,IAAI,IAAI,CAAC,IAAI;oBAAE,OAAO,IAAI,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAyB;QACrC,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,IAAI,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;gBACzC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC;QACF,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,SAAS,CAAC,MAAmB;QACnC,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;IACzC,CAAC;IAEO,SAAS,CAAC,GAAY;QAC5B,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF"}
|
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
import type { VoicePipelineConfig } from '../types.js';
|
|
2
|
-
import { PipelineSession } from './pipeline-session.js';
|
|
2
|
+
import { PipelineSession, type PipelineSessionOptions } from './pipeline-session.js';
|
|
3
|
+
export interface VoicePipelineResult {
|
|
4
|
+
transcript: string;
|
|
5
|
+
response: string;
|
|
6
|
+
audio: Buffer;
|
|
7
|
+
}
|
|
3
8
|
export declare class VoicePipeline {
|
|
4
9
|
private readonly config;
|
|
5
10
|
constructor(config: VoicePipelineConfig);
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
createSession(): PipelineSession;
|
|
11
|
+
/**
|
|
12
|
+
* One-shot STT -> Agent -> TTS. Silent input (empty transcript) skips the agent and TTS
|
|
13
|
+
* and returns empty strings with an empty audio buffer.
|
|
14
|
+
*/
|
|
15
|
+
process(audio: Buffer, options?: PipelineSessionOptions): Promise<VoicePipelineResult>;
|
|
16
|
+
createSession(options?: PipelineSessionOptions): PipelineSession;
|
|
12
17
|
}
|
|
13
18
|
//# sourceMappingURL=voice-pipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voice-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"voice-pipeline.d.ts","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAErF,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAEjC,MAAM,EAAE,mBAAmB;IAIvC;;;OAGG;IACG,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAkBhG,aAAa,CAAC,OAAO,CAAC,EAAE,sBAAsB,GAAG,eAAe;CAGjE"}
|
|
@@ -4,14 +4,24 @@ export class VoicePipeline {
|
|
|
4
4
|
constructor(config) {
|
|
5
5
|
this.config = config;
|
|
6
6
|
}
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* One-shot STT -> Agent -> TTS. Silent input (empty transcript) skips the agent and TTS
|
|
9
|
+
* and returns empty strings with an empty audio buffer.
|
|
10
|
+
*/
|
|
11
|
+
async process(audio, options = {}) {
|
|
8
12
|
const { text: transcript } = await this.config.stt.transcribe(audio);
|
|
9
|
-
|
|
13
|
+
if (transcript.trim().length === 0) {
|
|
14
|
+
return { transcript, response: '', audio: Buffer.alloc(0) };
|
|
15
|
+
}
|
|
16
|
+
const { content: response } = await this.config.agent.run(transcript, options.sessionId ? { sessionId: options.sessionId } : undefined);
|
|
17
|
+
if (response.trim().length === 0) {
|
|
18
|
+
return { transcript, response, audio: Buffer.alloc(0) };
|
|
19
|
+
}
|
|
10
20
|
const outputAudio = await this.config.tts.synthesize(response);
|
|
11
21
|
return { transcript, response, audio: outputAudio };
|
|
12
22
|
}
|
|
13
|
-
createSession() {
|
|
14
|
-
return new PipelineSession(this.config);
|
|
23
|
+
createSession(options) {
|
|
24
|
+
return new PipelineSession(this.config, options);
|
|
15
25
|
}
|
|
16
26
|
}
|
|
17
27
|
//# sourceMappingURL=voice-pipeline.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voice-pipeline.js","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"voice-pipeline.js","sourceRoot":"","sources":["../../src/pipeline/voice-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AAQrF,MAAM,OAAO,aAAa;IACP,MAAM,CAAsB;IAE7C,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,UAAkC,EAAE;QAC/D,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CACvD,UAAU,EACV,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CACjE,CAAC;QACF,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC/D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IACtD,CAAC;IAED,aAAa,CAAC,OAAgC;QAC5C,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;CACF"}
|
|
@@ -10,25 +10,37 @@ interface GeminiRealtimeEvents {
|
|
|
10
10
|
turn_end: [];
|
|
11
11
|
error: [error: Error];
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* Gemini Live API adapter. Input audio is PCM16 mono at 16kHz,
|
|
15
|
+
* output audio is PCM16 mono at 24kHz.
|
|
16
|
+
*/
|
|
13
17
|
export declare class GeminiRealtimeAdapter extends EventEmitter<GeminiRealtimeEvents> {
|
|
14
18
|
private readonly config;
|
|
15
19
|
private readonly model;
|
|
16
20
|
private ws;
|
|
21
|
+
private connected;
|
|
22
|
+
private modelTurnActive;
|
|
17
23
|
private interrupting;
|
|
24
|
+
private userTranscript;
|
|
25
|
+
private assistantTranscript;
|
|
26
|
+
private assistantText;
|
|
18
27
|
constructor(config: RealtimeSessionConfig);
|
|
28
|
+
get isConnected(): boolean;
|
|
19
29
|
connect(): Promise<void>;
|
|
20
30
|
pushAudio(chunk: Buffer): void;
|
|
21
31
|
sendText(text: string): void;
|
|
22
32
|
/**
|
|
23
33
|
* Gemini Live barge-in is driven by incoming user audio, not a control message.
|
|
24
|
-
*
|
|
25
|
-
* turn completes
|
|
34
|
+
* While a model turn is in progress this drops the remaining model audio until
|
|
35
|
+
* the turn completes, simulating an interruption on the consumer side.
|
|
26
36
|
*/
|
|
27
37
|
interrupt(): void;
|
|
28
38
|
close(): void;
|
|
29
39
|
private sendSetup;
|
|
30
|
-
private handleMessage;
|
|
31
40
|
private handleServerContent;
|
|
41
|
+
private flushUserTranscript;
|
|
42
|
+
private flushAssistantTranscript;
|
|
43
|
+
private resetTurn;
|
|
32
44
|
private handleToolCalls;
|
|
33
45
|
private send;
|
|
34
46
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini-realtime.d.ts","sourceRoot":"","sources":["../../src/realtime/gemini-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AASzD,UAAU,oBAAoB;IAC5B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAmCD;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,YAAY,CAAC,oBAAoB,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,cAAc,CAAM;IAC5B,OAAO,CAAC,mBAAmB,CAAM;IACjC,OAAO,CAAC,aAAa,CAAM;gBAEf,MAAM,EAAE,qBAAqB;IAMzC,IAAI,WAAW,IAAI,OAAO,CAEzB;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAmF9B,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAQ9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5B;;;;OAIG;IACH,SAAS,IAAI,IAAI;IAMjB,KAAK,IAAI,IAAI;IAWb,OAAO,CAAC,SAAS;IAkCjB,OAAO,CAAC,mBAAmB;IA2C3B,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,wBAAwB;IAOhC,OAAO,CAAC,SAAS;YAQH,eAAe;IA8B7B,OAAO,CAAC,IAAI;CAMb"}
|