@just-every/ensemble 0.2.187 → 0.2.188
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/ensemble_embed.cjs +94 -40
- package/dist/cjs/core/ensemble_embed.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_embed.js.map +1 -1
- package/dist/cjs/core/ensemble_image.cjs +47 -8
- package/dist/cjs/core/ensemble_image.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_image.js.map +1 -1
- package/dist/cjs/core/ensemble_listen.cjs +151 -74
- package/dist/cjs/core/ensemble_listen.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_listen.js.map +1 -1
- package/dist/cjs/core/ensemble_live.cjs +193 -40
- package/dist/cjs/core/ensemble_live.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_live.js.map +1 -1
- package/dist/cjs/core/ensemble_request.cjs +107 -11
- package/dist/cjs/core/ensemble_request.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_request.js.map +1 -1
- package/dist/cjs/core/ensemble_voice.cjs +190 -146
- package/dist/cjs/core/ensemble_voice.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_voice.js.map +1 -1
- package/dist/cjs/index.cjs +1 -0
- package/dist/cjs/index.d.ts +1 -0
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/types.d.ts +13 -0
- package/dist/cjs/types/types.d.ts.map +1 -1
- package/dist/cjs/utils/trace_context.cjs +96 -0
- package/dist/cjs/utils/trace_context.d.ts +20 -0
- package/dist/cjs/utils/trace_context.d.ts.map +1 -0
- package/dist/cjs/utils/trace_context.js.map +1 -0
- package/dist/cjs/utils/trace_logger.cjs +66 -0
- package/dist/cjs/utils/trace_logger.d.ts +12 -0
- package/dist/cjs/utils/trace_logger.d.ts.map +1 -0
- package/dist/cjs/utils/trace_logger.js.map +1 -0
- package/dist/core/ensemble_embed.d.ts.map +1 -1
- package/dist/core/ensemble_embed.js +94 -40
- package/dist/core/ensemble_embed.js.map +1 -1
- package/dist/core/ensemble_image.d.ts.map +1 -1
- package/dist/core/ensemble_image.js +47 -8
- package/dist/core/ensemble_image.js.map +1 -1
- package/dist/core/ensemble_listen.d.ts.map +1 -1
- package/dist/core/ensemble_listen.js +151 -74
- package/dist/core/ensemble_listen.js.map +1 -1
- package/dist/core/ensemble_live.d.ts.map +1 -1
- package/dist/core/ensemble_live.js +193 -40
- package/dist/core/ensemble_live.js.map +1 -1
- package/dist/core/ensemble_request.d.ts.map +1 -1
- package/dist/core/ensemble_request.js +107 -11
- package/dist/core/ensemble_request.js.map +1 -1
- package/dist/core/ensemble_voice.d.ts.map +1 -1
- package/dist/core/ensemble_voice.js +190 -146
- package/dist/core/ensemble_voice.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types/types.d.ts +13 -0
- package/dist/types/types.d.ts.map +1 -1
- package/dist/utils/trace_context.d.ts +20 -0
- package/dist/utils/trace_context.d.ts.map +1 -0
- package/dist/utils/trace_context.js +91 -0
- package/dist/utils/trace_context.js.map +1 -0
- package/dist/utils/trace_logger.d.ts +12 -0
- package/dist/utils/trace_logger.d.ts.map +1 -0
- package/dist/utils/trace_logger.js +58 -0
- package/dist/utils/trace_logger.js.map +1 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ensemble_listen.d.ts","sourceRoot":"","sources":["../../core/ensemble_listen.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,eAAe,EACf,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACrB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"ensemble_listen.d.ts","sourceRoot":"","sources":["../../core/ensemble_listen.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,eAAe,EACf,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACrB,MAAM,mBAAmB,CAAC;AAM3B,YAAY,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,CAAC;AAsEhF,wBAAuB,cAAc,CACjC,WAAW,EAAE,wBAAwB,EACrC,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE,iBAAsB,GAChC,cAAc,CAAC,kBAAkB,CAAC,CAwMpC;AAUD,wBAAgB,gCAAgC,CAC5C,WAAW,EAAE,WAAW,EACxB,YAAY,CAAC,EAAE,YAAY,GAC5B,cAAc,CAAC,UAAU,CAAC,CA4B5B"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { getModelFromAgent, getModelProvider } from '../model_providers/model_provider.js';
|
|
2
|
+
import { createTraceContext } from '../utils/trace_context.js';
|
|
3
|
+
import { randomUUID } from 'crypto';
|
|
2
4
|
function normalizeAudioSource(source) {
|
|
3
5
|
if (source instanceof ReadableStream) {
|
|
4
6
|
return source;
|
|
@@ -34,101 +36,176 @@ function normalizeAudioSource(source) {
|
|
|
34
36
|
throw new Error(`Unsupported audio source type: ${typeof source}`);
|
|
35
37
|
}
|
|
36
38
|
export async function* ensembleListen(audioSource, agent, options = {}) {
|
|
39
|
+
const trace = createTraceContext(agent, 'transcription');
|
|
40
|
+
const requestId = randomUUID();
|
|
41
|
+
let requestStarted = false;
|
|
42
|
+
let turnStatus = 'completed';
|
|
43
|
+
let requestStatus = 'completed';
|
|
44
|
+
let requestError;
|
|
45
|
+
let transcriptionDuration;
|
|
46
|
+
let finalTranscript;
|
|
37
47
|
const streamOptions = { ...options, stream: true };
|
|
38
48
|
const audioFormat = options.audioFormat || {
|
|
39
49
|
sampleRate: 16000,
|
|
40
50
|
channels: 1,
|
|
41
51
|
encoding: 'pcm',
|
|
42
52
|
};
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
timestamp: new Date().toISOString(),
|
|
59
|
-
error: `Failed to initialize provider for model ${model}: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
|
60
|
-
};
|
|
61
|
-
yield errorEvent;
|
|
62
|
-
return;
|
|
63
|
-
}
|
|
64
|
-
if (!provider.createTranscription) {
|
|
65
|
-
const errorEvent = {
|
|
66
|
-
type: 'error',
|
|
67
|
-
timestamp: new Date().toISOString(),
|
|
68
|
-
error: `Provider for model ${model} does not support transcription`,
|
|
69
|
-
};
|
|
70
|
-
yield errorEvent;
|
|
71
|
-
return;
|
|
72
|
-
}
|
|
73
|
-
let audioStream;
|
|
53
|
+
const audioSourceType = audioSource instanceof ReadableStream
|
|
54
|
+
? 'readable_stream'
|
|
55
|
+
: audioSource instanceof ArrayBuffer
|
|
56
|
+
? 'array_buffer'
|
|
57
|
+
: audioSource instanceof Uint8Array
|
|
58
|
+
? 'uint8array'
|
|
59
|
+
: typeof audioSource === 'function'
|
|
60
|
+
? 'factory'
|
|
61
|
+
: typeof audioSource === 'object' && audioSource !== null && Symbol.asyncIterator in audioSource
|
|
62
|
+
? 'async_iterable'
|
|
63
|
+
: typeof audioSource;
|
|
64
|
+
await trace.emitTurnStart({
|
|
65
|
+
audio_source_type: audioSourceType,
|
|
66
|
+
options: streamOptions,
|
|
67
|
+
});
|
|
74
68
|
try {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
69
|
+
const model = await getModelFromAgent(agent, 'transcription');
|
|
70
|
+
await trace.emitRequestStart(requestId, {
|
|
71
|
+
agent_id: agent.agent_id,
|
|
72
|
+
model,
|
|
73
|
+
payload: {
|
|
74
|
+
audio_source_type: audioSourceType,
|
|
75
|
+
options: streamOptions,
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
requestStarted = true;
|
|
79
|
+
const startEvent = {
|
|
80
|
+
type: 'transcription_start',
|
|
80
81
|
timestamp: new Date().toISOString(),
|
|
81
|
-
|
|
82
|
+
format: audioFormat.encoding || 'pcm',
|
|
83
|
+
audioFormat: audioFormat,
|
|
82
84
|
};
|
|
83
|
-
yield
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
85
|
+
yield startEvent;
|
|
86
|
+
let provider;
|
|
87
|
+
try {
|
|
88
|
+
provider = getModelProvider(model);
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
requestStatus = 'error';
|
|
92
|
+
turnStatus = 'error';
|
|
93
|
+
requestError = `Failed to initialize provider for model ${model}: ${error instanceof Error ? error.message : 'Unknown error'}`;
|
|
94
|
+
const errorEvent = {
|
|
95
|
+
type: 'error',
|
|
96
|
+
timestamp: new Date().toISOString(),
|
|
97
|
+
error: requestError,
|
|
98
|
+
};
|
|
99
|
+
yield errorEvent;
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (!provider.createTranscription) {
|
|
103
|
+
requestStatus = 'error';
|
|
104
|
+
turnStatus = 'error';
|
|
105
|
+
requestError = `Provider for model ${model} does not support transcription`;
|
|
106
|
+
const errorEvent = {
|
|
107
|
+
type: 'error',
|
|
108
|
+
timestamp: new Date().toISOString(),
|
|
109
|
+
error: requestError,
|
|
110
|
+
};
|
|
111
|
+
yield errorEvent;
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
let audioStream;
|
|
115
|
+
try {
|
|
116
|
+
audioStream = normalizeAudioSource(audioSource);
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
requestStatus = 'error';
|
|
120
|
+
turnStatus = 'error';
|
|
121
|
+
requestError = `Failed to normalize audio source: ${error instanceof Error ? error.message : 'Unknown error'}`;
|
|
122
|
+
const errorEvent = {
|
|
123
|
+
type: 'error',
|
|
124
|
+
timestamp: new Date().toISOString(),
|
|
125
|
+
error: requestError,
|
|
126
|
+
};
|
|
127
|
+
yield errorEvent;
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
const startTime = Date.now();
|
|
131
|
+
let fullTranscript = '';
|
|
132
|
+
let currentTurnText = '';
|
|
133
|
+
const allTurns = [];
|
|
134
|
+
try {
|
|
135
|
+
const transcriptionGenerator = provider.createTranscription(audioStream, agent, model, streamOptions);
|
|
136
|
+
for await (const event of transcriptionGenerator) {
|
|
137
|
+
if (event.type === 'transcription_turn_delta' && event.delta) {
|
|
138
|
+
fullTranscript += event.delta;
|
|
139
|
+
currentTurnText += event.delta;
|
|
140
|
+
}
|
|
141
|
+
if (event.type === 'transcription_turn_complete') {
|
|
142
|
+
const turnEvent = {
|
|
143
|
+
...event,
|
|
144
|
+
text: currentTurnText,
|
|
145
|
+
};
|
|
146
|
+
yield turnEvent;
|
|
147
|
+
if (currentTurnText.trim()) {
|
|
148
|
+
allTurns.push(currentTurnText.trim());
|
|
149
|
+
}
|
|
150
|
+
currentTurnText = '';
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
yield event;
|
|
105
154
|
}
|
|
106
|
-
currentTurnText = '';
|
|
107
155
|
}
|
|
108
|
-
|
|
109
|
-
|
|
156
|
+
if (currentTurnText.trim()) {
|
|
157
|
+
allTurns.push(currentTurnText.trim());
|
|
110
158
|
}
|
|
159
|
+
const duration = (Date.now() - startTime) / 1000;
|
|
160
|
+
const transcript = allTurns.length > 0 ? allTurns.join('\n') : fullTranscript;
|
|
161
|
+
transcriptionDuration = duration;
|
|
162
|
+
finalTranscript = transcript;
|
|
163
|
+
const completeEvent = {
|
|
164
|
+
type: 'transcription_complete',
|
|
165
|
+
timestamp: new Date().toISOString(),
|
|
166
|
+
text: transcript,
|
|
167
|
+
duration: duration,
|
|
168
|
+
};
|
|
169
|
+
yield completeEvent;
|
|
111
170
|
}
|
|
112
|
-
|
|
113
|
-
|
|
171
|
+
catch (error) {
|
|
172
|
+
requestStatus = 'error';
|
|
173
|
+
turnStatus = 'error';
|
|
174
|
+
requestError = error instanceof Error ? error.message : 'Transcription failed';
|
|
175
|
+
console.error('[ensembleListen] Error during transcription:', error);
|
|
176
|
+
const errorEvent = {
|
|
177
|
+
type: 'error',
|
|
178
|
+
timestamp: new Date().toISOString(),
|
|
179
|
+
error: requestError,
|
|
180
|
+
};
|
|
181
|
+
yield errorEvent;
|
|
114
182
|
}
|
|
115
|
-
const duration = (Date.now() - startTime) / 1000;
|
|
116
|
-
const completeEvent = {
|
|
117
|
-
type: 'transcription_complete',
|
|
118
|
-
timestamp: new Date().toISOString(),
|
|
119
|
-
text: allTurns.length > 0 ? allTurns.join('\n') : fullTranscript,
|
|
120
|
-
duration: duration,
|
|
121
|
-
};
|
|
122
|
-
yield completeEvent;
|
|
123
183
|
}
|
|
124
184
|
catch (error) {
|
|
125
|
-
|
|
185
|
+
requestStatus = 'error';
|
|
186
|
+
turnStatus = 'error';
|
|
187
|
+
requestError = error instanceof Error ? error.message : String(error);
|
|
126
188
|
const errorEvent = {
|
|
127
189
|
type: 'error',
|
|
128
190
|
timestamp: new Date().toISOString(),
|
|
129
|
-
error:
|
|
191
|
+
error: requestError,
|
|
130
192
|
};
|
|
131
193
|
yield errorEvent;
|
|
194
|
+
throw error;
|
|
195
|
+
}
|
|
196
|
+
finally {
|
|
197
|
+
if (requestStarted) {
|
|
198
|
+
await trace.emitRequestEnd(requestId, {
|
|
199
|
+
status: requestStatus,
|
|
200
|
+
error: requestError,
|
|
201
|
+
duration: transcriptionDuration,
|
|
202
|
+
final_response: finalTranscript,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
await trace.emitTurnEnd(turnStatus, turnStatus === 'completed' ? 'completed' : 'exception', {
|
|
206
|
+
error: requestError,
|
|
207
|
+
final_response: finalTranscript,
|
|
208
|
+
});
|
|
132
209
|
}
|
|
133
210
|
}
|
|
134
211
|
export function createAudioStreamFromMediaStream(mediaStream, audioContext) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ensemble_listen.js","sourceRoot":"","sources":["../../core/ensemble_listen.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAsB,MAAM,sCAAsC,CAAC;
|
|
1
|
+
{"version":3,"file":"ensemble_listen.js","sourceRoot":"","sources":["../../core/ensemble_listen.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAsB,MAAM,sCAAsC,CAAC;AAC/G,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAQpC,SAAS,oBAAoB,CAAC,MAAgC;IAE1D,IAAI,MAAM,YAAY,cAAc,EAAE,CAAC;QACnC,OAAO,MAAM,CAAC;IAClB,CAAC;IAGD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC;QAClF,OAAO,IAAI,cAAc,CAAC;YACtB,KAAK,CAAC,KAAK,CAAC,UAAU;gBAClB,IAAI,CAAC;oBACD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAmC,EAAE,CAAC;wBAC5D,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC9B,CAAC;oBACD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACvB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAGD,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC;QAC1B,OAAO,oBAAoB,CAAC,QAAoC,CAAC,CAAC;IACtE,CAAC;IAGD,IAAI,MAAM,YAAY,WAAW,IAAI,MAAM,YAAY,UAAU,EAAE,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7E,OAAO,IAAI,cAAc,CAAC;YACtB,KAAK,CAAC,UAAU;gBACZ,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACzB,UAAU,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,MAAM,EAAE,CAAC,CAAC;AACvE,CAAC;AAyBD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACjC,WAAqC,EACrC,KAAsB,EACtB,UAA6B,EAAE;IAE/B,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,UAAU,GAA0B,WAAW,CAAC;IACpD,IAAI,aAAa,GAAG,WAAW,CAAC;IAChC,IAAI,YAAgC,CAAC;IACrC,IAAI,qBAAyC,CAAC;IAC9C,IAAI,eAAmC,CAAC;IAGxC,MAAM,aAAa,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAGnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI;QACvC,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,KAAc;KAC3B,CAAC;IAEF,MAAM,eAAe,GACjB,WAAW,YAAY,cAAc;QACjC,CAAC,CAAC,iBAAiB;QACnB,CAAC,CAAC,WAAW,YAAY,WAAW;YAClC,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,YAAY,UAAU;gBACjC,CAAC,CAAC,YAAY;gBACd,CAAC,CAAC,OAAO,WAAW,KAAK,UAAU;oBACjC,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,KAAK,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,WAAW;wBAC9F,CAAC,CAAC,gBAAgB;wBAClB,CAAC,CAAC,OAAO,WAAW,CAAC;IAErC,MAAM,KAAK,CAAC,aAAa,CAAC;QACtB,iBAAiB,EAAE,eAAe;QAClC,OAAO,EAAE,aAAa;KACzB,CAAC,CAAC;IAEH,IAAI,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,gBAAgB,CAAC,SAAS,EAAE;YACpC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK;YACL,OAAO,EAAE;gBACL,iBAAiB,EAAE,eAAe;gBAClC,OAAO,EAAE,aAAa;aACzB;SACJ,CAAC,CAAC;QACH,cAAc,GAAG,IAAI,CAAC;QAGtB,MAAM,UAAU,GAAuB;YACnC,IAAI,EAAE,qBAAqB;YAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,WAAW,CAAC,QAAQ,IAAI,KAAK;YACrC,WAAW,EAAE,WAAW;SAC3B,CAAC;QACF,MAAM,UAAU,CAAC;QAGjB,IAAI,QAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,aAAa,GAAG,OAAO,CAAC;YACxB,UAAU,GAAG,OAAO,CAAC;YACrB,YAAY,GAAG,2CAA2C,KAAK,KAC3D,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAC7C,EAAE,CAAC;YACH,MAAM,UAAU,GAAuB;gBACnC,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,YAAY;aACtB,CAAC;YACF,MAAM,UAAU,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAChC,aAAa,GAAG,OAAO,CAAC;YACxB,UAAU,GAAG,OAAO,CAAC;YACrB,YAAY,GAAG,sBAAsB,KAAK,iCAAiC,CAAC;YAC5E,MAAM,UAAU,GAAuB;gBACnC,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,YAAY;aACtB,CAAC;YACF,MAAM,UAAU,CAAC;YACjB,OAAO;QACX,CAAC;QAGD,IAAI,WAAuC,CAAC;QAC5C,IAAI,CAAC;YACD,WAAW,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,aAAa,GAAG,OAAO,CAAC;YACxB,UAAU,GAAG,OAAO,CAAC;YACrB,YAAY,GAAG,qCAAqC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC;YAC/G,MAAM,UAAU,GAAuB;gBACnC,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,YAAY;aACtB,CAAC;YACF,MAAM,UAAU,CAAC;YACjB,OAAO;QACX,CAAC;QAGD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,IAAI,eAAe,GAAG,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IAAI,CAAC;YAED,MAAM,sBAAsB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YAEtG,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,sBAAsB,EAAE,CAAC;gBAE/C,IAAI,KAAK,CAAC,IAAI,KAAK,0BAA0B,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC3D,cAAc,IAAI,KAAK,CAAC,KAAK,CAAC;oBAC9B,eAAe,IAAI,KAAK,CAAC,KAAK,CAAC;gBACnC,CAAC;gBAGD,IAAI,KAAK,CAAC,IAAI,KAAK,6BAA6B,EAAE,CAAC;oBAE/C,MAAM,SAAS,GAAuB;wBAClC,GAAG,KAAK;wBACR,IAAI,EAAE,eAAe;qBACxB,CAAC;oBACF,MAAM,SAAS,CAAC;oBAGhB,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;wBACzB,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC1C,CAAC;oBACD,eAAe,GAAG,EAAE,CAAC;gBACzB,CAAC;qBAAM,CAAC;oBAEJ,MAAM,KAAK,CAAC;gBAChB,CAAC;YACL,CAAC;YAGD,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1C,CAAC;YAGD,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;YACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YAC9E,qBAAqB,GAAG,QAAQ,CAAC;YACjC,eAAe,GAAG,UAAU,CAAC;YAC7B,MAAM,aAAa,GAAuB;gBACtC,IAAI,EAAE,wBAAwB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE,QAAQ;aACrB,CAAC;YACF,MAAM,aAAa,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,aAAa,GAAG,OAAO,CAAC;YACxB,UAAU,GAAG,OAAO,CAAC;YACrB,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC;YAC/E,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,MAAM,UAAU,GAAuB;gBACnC,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,YAAY;aACtB,CAAC;YACF,MAAM,UAAU,CAAC;QACrB,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,aAAa,GAAG,OAAO,CAAC;QACxB,UAAU,GAAG,OAAO,CAAC;QACrB,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,UAAU,GAAuB;YACnC,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK,EAAE,YAAY;SACtB,CAAC;QACF,MAAM,UAAU,CAAC;QACjB,MAAM,KAAK,CAAC;IAChB,CAAC;YAAS,CAAC;QACP,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE;gBAClC,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,qBAAqB;gBAC/B,cAAc,EAAE,eAAe;aAClC,CAAC,CAAC;QACP,CAAC;QACD,MAAM,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,UAAU,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,EAAE;YACxF,KAAK,EAAE,YAAY;YACnB,cAAc,EAAE,eAAe;SAClC,CAAC,CAAC;IACP,CAAC;AACL,CAAC;AAUD,MAAM,UAAU,gCAAgC,CAC5C,WAAwB,EACxB,YAA2B;IAE3B,MAAM,GAAG,GAAG,YAAY,IAAI,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,GAAG,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAExD,OAAO,IAAI,cAAc,CAAC;QACtB,KAAK,CAAC,UAAU;YACZ,SAAS,CAAC,cAAc,GAAG,CAAC,CAAC,EAAE;gBAC3B,MAAM,YAAY,GAAG,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;gBACrD,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAGvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrD,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBACpD,CAAC;gBAED,UAAU,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1D,CAAC,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC1B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC;QACD,MAAM;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,MAAM,CAAC,UAAU,EAAE,CAAC;QACxB,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ensemble_live.d.ts","sourceRoot":"","sources":["../../core/ensemble_live.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,UAAU,EACV,WAAW,EACX,SAAS,EAET,eAAe,EAWlB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"ensemble_live.d.ts","sourceRoot":"","sources":["../../core/ensemble_live.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,UAAU,EACV,WAAW,EACX,SAAS,EAET,eAAe,EAWlB,MAAM,mBAAmB,CAAC;AAgC3B,wBAAuB,YAAY,CAC/B,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,EACtB,OAAO,CAAC,EAAE,WAAW,GACtB,cAAc,CAAC,SAAS,CAAC,CAiY3B;AAUD,wBAAuB,iBAAiB,CACpC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,EACtC,KAAK,EAAE,eAAe,EACtB,OAAO,CAAC,EAAE,WAAW,GAAG;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC/B,GACF,cAAc,CAAC,SAAS,CAAC,CA6L3B;AASD,wBAAsB,gBAAgB,CAClC,KAAK,EAAE,eAAe,EACtB,OAAO,CAAC,EAAE,WAAW,GACtB,OAAO,CAAC;IACP,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,SAAS,EAAE,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAC1C,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B,CAAC,CAkED"}
|
|
@@ -4,8 +4,12 @@ import { MessageHistory } from '../utils/message_history.js';
|
|
|
4
4
|
import { handleToolCall } from '../utils/tool_execution_manager.js';
|
|
5
5
|
import { processToolResult } from '../utils/tool_result_processor.js';
|
|
6
6
|
import { emitEvent } from '../utils/event_controller.js';
|
|
7
|
+
import { createTraceContext } from '../utils/trace_context.js';
|
|
8
|
+
import { randomUUID } from 'crypto';
|
|
7
9
|
export async function* ensembleLive(config, agent, options) {
|
|
8
10
|
const startTime = Date.now();
|
|
11
|
+
const trace = createTraceContext(agent, 'live_session');
|
|
12
|
+
const requestId = randomUUID();
|
|
9
13
|
let session = null;
|
|
10
14
|
let messageHistory = null;
|
|
11
15
|
let totalToolCalls = 0;
|
|
@@ -13,18 +17,42 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
13
17
|
let isSessionActive = true;
|
|
14
18
|
let totalCost = 0;
|
|
15
19
|
let totalTokens = 0;
|
|
20
|
+
let requestStarted = false;
|
|
21
|
+
let turnStatus = 'completed';
|
|
22
|
+
let turnEndReason = 'completed';
|
|
23
|
+
let requestStatus = 'completed';
|
|
24
|
+
let requestError;
|
|
25
|
+
let resolvedModel;
|
|
26
|
+
let resolvedProviderId;
|
|
27
|
+
await trace.emitTurnStart({
|
|
28
|
+
config,
|
|
29
|
+
options,
|
|
30
|
+
});
|
|
16
31
|
try {
|
|
17
32
|
const model = await getModelFromAgent(agent);
|
|
18
33
|
if (!model) {
|
|
19
34
|
throw new Error('No model specified in agent configuration');
|
|
20
35
|
}
|
|
36
|
+
resolvedModel = model;
|
|
21
37
|
const provider = getModelProvider(model);
|
|
22
38
|
if (!provider) {
|
|
23
39
|
throw new Error(`No provider found for model: ${model}`);
|
|
24
40
|
}
|
|
41
|
+
resolvedProviderId = provider.provider_id;
|
|
25
42
|
if (!provider.createLiveSession) {
|
|
26
43
|
throw new Error(`Provider ${provider.provider_id} does not support Live API`);
|
|
27
44
|
}
|
|
45
|
+
await trace.emitRequestStart(requestId, {
|
|
46
|
+
agent_id: agent.agent_id,
|
|
47
|
+
provider: provider.provider_id,
|
|
48
|
+
model,
|
|
49
|
+
payload: {
|
|
50
|
+
config,
|
|
51
|
+
options,
|
|
52
|
+
history_count: options?.messageHistory?.length ?? 0,
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
requestStarted = true;
|
|
28
56
|
if (options?.messageHistory) {
|
|
29
57
|
messageHistory = new MessageHistory();
|
|
30
58
|
for (const message of options.messageHistory) {
|
|
@@ -84,10 +112,14 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
84
112
|
const maxToolCalls = options?.maxToolCalls ?? agent.maxToolCalls ?? 200;
|
|
85
113
|
const maxToolCallRoundsPerTurn = options?.maxToolCallRoundsPerTurn ?? agent.maxToolCallRoundsPerTurn ?? Infinity;
|
|
86
114
|
if (totalToolCalls >= maxToolCalls) {
|
|
115
|
+
turnStatus = 'error';
|
|
116
|
+
turnEndReason = 'max_tool_calls_exceeded';
|
|
117
|
+
requestStatus = 'error';
|
|
118
|
+
requestError = `Maximum tool calls (${maxToolCalls}) exceeded`;
|
|
87
119
|
const errorEvent = {
|
|
88
120
|
type: 'error',
|
|
89
121
|
timestamp: new Date().toISOString(),
|
|
90
|
-
error:
|
|
122
|
+
error: requestError,
|
|
91
123
|
code: 'MAX_TOOL_CALLS_EXCEEDED',
|
|
92
124
|
recoverable: false,
|
|
93
125
|
};
|
|
@@ -106,6 +138,11 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
106
138
|
continue;
|
|
107
139
|
}
|
|
108
140
|
for (const toolCall of toolCallEvent.toolCalls) {
|
|
141
|
+
await trace.emitToolStart(requestId, toolCall.id, {
|
|
142
|
+
tool_name: toolCall.function.name,
|
|
143
|
+
call_id: toolCall.call_id || toolCall.id,
|
|
144
|
+
arguments: toolCall.function.arguments,
|
|
145
|
+
});
|
|
109
146
|
const toolStartEvent = {
|
|
110
147
|
type: 'tool_start',
|
|
111
148
|
timestamp: new Date().toISOString(),
|
|
@@ -129,6 +166,11 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
129
166
|
toolResults.push(toolCallResult);
|
|
130
167
|
totalToolCalls++;
|
|
131
168
|
currentTurnToolCalls++;
|
|
169
|
+
await trace.emitToolDone(requestId, toolCall.id, {
|
|
170
|
+
tool_name: toolCall.function.name,
|
|
171
|
+
call_id: toolCallResult.call_id,
|
|
172
|
+
output: toolCallResult.output,
|
|
173
|
+
});
|
|
132
174
|
const toolResultEvent = {
|
|
133
175
|
type: 'tool_result',
|
|
134
176
|
timestamp: new Date().toISOString(),
|
|
@@ -148,6 +190,11 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
148
190
|
error: errorMessage,
|
|
149
191
|
};
|
|
150
192
|
toolResults.push(toolCallResult);
|
|
193
|
+
await trace.emitToolDone(requestId, toolCall.id, {
|
|
194
|
+
tool_name: toolCall.function.name,
|
|
195
|
+
call_id: toolCallResult.call_id,
|
|
196
|
+
error: errorMessage,
|
|
197
|
+
});
|
|
151
198
|
const errorEvent = {
|
|
152
199
|
type: 'error',
|
|
153
200
|
timestamp: new Date().toISOString(),
|
|
@@ -199,6 +246,10 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
199
246
|
}
|
|
200
247
|
catch (error) {
|
|
201
248
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
249
|
+
turnStatus = 'error';
|
|
250
|
+
turnEndReason = 'exception';
|
|
251
|
+
requestStatus = 'error';
|
|
252
|
+
requestError = errorMessage;
|
|
202
253
|
const errorEvent = {
|
|
203
254
|
type: 'error',
|
|
204
255
|
timestamp: new Date().toISOString(),
|
|
@@ -215,10 +266,33 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
215
266
|
}
|
|
216
267
|
isSessionActive = false;
|
|
217
268
|
const duration = Date.now() - startTime;
|
|
269
|
+
if (requestStarted) {
|
|
270
|
+
await trace.emitRequestEnd(requestId, {
|
|
271
|
+
status: requestStatus,
|
|
272
|
+
error: requestError,
|
|
273
|
+
duration_ms: duration,
|
|
274
|
+
total_tokens: totalTokens,
|
|
275
|
+
total_cost: totalCost > 0 ? totalCost : undefined,
|
|
276
|
+
total_tool_calls: totalToolCalls,
|
|
277
|
+
session_id: session?.sessionId,
|
|
278
|
+
model: resolvedModel,
|
|
279
|
+
provider: resolvedProviderId,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
await trace.emitTurnEnd(turnStatus, turnEndReason, {
|
|
283
|
+
error: requestError,
|
|
284
|
+
duration_ms: duration,
|
|
285
|
+
total_tokens: totalTokens,
|
|
286
|
+
total_cost: totalCost > 0 ? totalCost : undefined,
|
|
287
|
+
total_tool_calls: totalToolCalls,
|
|
288
|
+
session_id: session?.sessionId,
|
|
289
|
+
model: resolvedModel,
|
|
290
|
+
provider: resolvedProviderId,
|
|
291
|
+
});
|
|
218
292
|
const endEvent = {
|
|
219
293
|
type: 'live_end',
|
|
220
294
|
timestamp: new Date().toISOString(),
|
|
221
|
-
reason:
|
|
295
|
+
reason: turnStatus === 'completed' ? 'completed' : 'error',
|
|
222
296
|
duration,
|
|
223
297
|
totalTokens,
|
|
224
298
|
totalCost: totalCost > 0 ? totalCost : undefined,
|
|
@@ -239,6 +313,16 @@ export async function* ensembleLive(config, agent, options) {
|
|
|
239
313
|
}
|
|
240
314
|
}
|
|
241
315
|
export async function* ensembleLiveAudio(audioSource, agent, options) {
|
|
316
|
+
const trace = createTraceContext(agent, 'live_audio_session');
|
|
317
|
+
const requestId = randomUUID();
|
|
318
|
+
let requestStarted = false;
|
|
319
|
+
let turnStatus = 'completed';
|
|
320
|
+
let turnEndReason = 'completed';
|
|
321
|
+
let requestStatus = 'completed';
|
|
322
|
+
let requestError;
|
|
323
|
+
let totalCost = 0;
|
|
324
|
+
let totalTokens = 0;
|
|
325
|
+
const startTime = Date.now();
|
|
242
326
|
const config = {
|
|
243
327
|
responseModalities: ['AUDIO'],
|
|
244
328
|
speechConfig: options?.voice
|
|
@@ -260,45 +344,66 @@ export async function* ensembleLiveAudio(audioSource, agent, options) {
|
|
|
260
344
|
proactiveAudio: true,
|
|
261
345
|
};
|
|
262
346
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
throw new Error(`Provider does not support Live API for model: ${model}`);
|
|
272
|
-
}
|
|
273
|
-
console.log('[ensembleLiveAudio] Creating live session...');
|
|
274
|
-
const session = await provider.createLiveSession(config, agent, model, options);
|
|
275
|
-
console.log('[ensembleLiveAudio] Session created:', session.sessionId);
|
|
347
|
+
await trace.emitTurnStart({
|
|
348
|
+
config,
|
|
349
|
+
options,
|
|
350
|
+
audio_source_type: 'async_iterable',
|
|
351
|
+
});
|
|
352
|
+
let session = null;
|
|
353
|
+
let model;
|
|
354
|
+
let providerId;
|
|
276
355
|
let audioChunkCount = 0;
|
|
277
356
|
let totalAudioBytes = 0;
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
break;
|
|
285
|
-
}
|
|
286
|
-
audioChunkCount++;
|
|
287
|
-
totalAudioBytes += chunk.length;
|
|
288
|
-
const base64Data = Buffer.from(chunk).toString('base64');
|
|
289
|
-
console.log(`[ensembleLiveAudio] Sending audio chunk ${audioChunkCount}, size: ${chunk.length} bytes, total: ${totalAudioBytes} bytes`);
|
|
290
|
-
await session.sendAudio({
|
|
291
|
-
data: base64Data,
|
|
292
|
-
mimeType: 'audio/pcm;rate=16000',
|
|
293
|
-
});
|
|
294
|
-
}
|
|
295
|
-
console.log(`[ensembleLiveAudio] Audio processing completed. Total chunks: ${audioChunkCount}, Total bytes: ${totalAudioBytes}`);
|
|
357
|
+
let audioProcessingTask = null;
|
|
358
|
+
try {
|
|
359
|
+
model = await getModelFromAgent(agent);
|
|
360
|
+
console.log('[ensembleLiveAudio] Using model:', model);
|
|
361
|
+
if (!model) {
|
|
362
|
+
throw new Error('No model specified in agent configuration');
|
|
296
363
|
}
|
|
297
|
-
|
|
298
|
-
|
|
364
|
+
const provider = getModelProvider(model);
|
|
365
|
+
providerId = provider?.provider_id;
|
|
366
|
+
console.log('[ensembleLiveAudio] Provider:', provider?.provider_id);
|
|
367
|
+
if (!provider || !provider.createLiveSession) {
|
|
368
|
+
throw new Error(`Provider does not support Live API for model: ${model}`);
|
|
299
369
|
}
|
|
300
|
-
|
|
301
|
-
|
|
370
|
+
await trace.emitRequestStart(requestId, {
|
|
371
|
+
agent_id: agent.agent_id,
|
|
372
|
+
provider: provider.provider_id,
|
|
373
|
+
model,
|
|
374
|
+
payload: {
|
|
375
|
+
config,
|
|
376
|
+
options,
|
|
377
|
+
audio_source_type: 'async_iterable',
|
|
378
|
+
},
|
|
379
|
+
});
|
|
380
|
+
requestStarted = true;
|
|
381
|
+
console.log('[ensembleLiveAudio] Creating live session...');
|
|
382
|
+
session = await provider.createLiveSession(config, agent, model, options);
|
|
383
|
+
console.log('[ensembleLiveAudio] Session created:', session.sessionId);
|
|
384
|
+
audioProcessingTask = (async () => {
|
|
385
|
+
try {
|
|
386
|
+
console.log('[ensembleLiveAudio] Starting audio processing task...');
|
|
387
|
+
for await (const chunk of audioSource) {
|
|
388
|
+
if (!session || !session.isActive()) {
|
|
389
|
+
console.log('[ensembleLiveAudio] Session inactive, stopping audio processing');
|
|
390
|
+
break;
|
|
391
|
+
}
|
|
392
|
+
audioChunkCount++;
|
|
393
|
+
totalAudioBytes += chunk.length;
|
|
394
|
+
const base64Data = Buffer.from(chunk).toString('base64');
|
|
395
|
+
console.log(`[ensembleLiveAudio] Sending audio chunk ${audioChunkCount}, size: ${chunk.length} bytes, total: ${totalAudioBytes} bytes`);
|
|
396
|
+
await session.sendAudio({
|
|
397
|
+
data: base64Data,
|
|
398
|
+
mimeType: 'audio/pcm;rate=16000',
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
console.log(`[ensembleLiveAudio] Audio processing completed. Total chunks: ${audioChunkCount}, Total bytes: ${totalAudioBytes}`);
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
console.error('[ensembleLiveAudio] Error processing audio:', error);
|
|
405
|
+
}
|
|
406
|
+
})();
|
|
302
407
|
yield {
|
|
303
408
|
type: 'live_start',
|
|
304
409
|
timestamp: new Date().toISOString(),
|
|
@@ -310,19 +415,63 @@ export async function* ensembleLiveAudio(audioSource, agent, options) {
|
|
|
310
415
|
for await (const event of session.getEventStream()) {
|
|
311
416
|
eventCount++;
|
|
312
417
|
console.log(`[ensembleLiveAudio] Event ${eventCount}:`, event.type);
|
|
418
|
+
if (event.type === 'cost_update') {
|
|
419
|
+
const costEvent = event;
|
|
420
|
+
if (costEvent.usage.totalCost) {
|
|
421
|
+
totalCost += costEvent.usage.totalCost;
|
|
422
|
+
}
|
|
423
|
+
if (costEvent.usage.totalTokens) {
|
|
424
|
+
totalTokens += costEvent.usage.totalTokens;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
313
427
|
yield event;
|
|
314
428
|
}
|
|
315
429
|
console.log(`[ensembleLiveAudio] Event processing completed. Total events: ${eventCount}`);
|
|
316
430
|
}
|
|
431
|
+
catch (error) {
|
|
432
|
+
turnStatus = 'error';
|
|
433
|
+
turnEndReason = 'exception';
|
|
434
|
+
requestStatus = 'error';
|
|
435
|
+
requestError = error instanceof Error ? error.message : String(error);
|
|
436
|
+
throw error;
|
|
437
|
+
}
|
|
317
438
|
finally {
|
|
318
|
-
|
|
319
|
-
|
|
439
|
+
if (audioProcessingTask) {
|
|
440
|
+
await audioProcessingTask;
|
|
441
|
+
}
|
|
442
|
+
if (session && session.isActive()) {
|
|
320
443
|
await session.close();
|
|
321
444
|
}
|
|
445
|
+
const duration = Date.now() - startTime;
|
|
446
|
+
if (requestStarted) {
|
|
447
|
+
await trace.emitRequestEnd(requestId, {
|
|
448
|
+
status: requestStatus,
|
|
449
|
+
error: requestError,
|
|
450
|
+
duration_ms: duration,
|
|
451
|
+
total_tokens: totalTokens,
|
|
452
|
+
total_cost: totalCost > 0 ? totalCost : undefined,
|
|
453
|
+
audio_chunks_sent: audioChunkCount,
|
|
454
|
+
audio_bytes_sent: totalAudioBytes,
|
|
455
|
+
session_id: session?.sessionId,
|
|
456
|
+
model,
|
|
457
|
+
provider: providerId,
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
await trace.emitTurnEnd(turnStatus, turnEndReason, {
|
|
461
|
+
error: requestError,
|
|
462
|
+
duration_ms: duration,
|
|
463
|
+
total_tokens: totalTokens,
|
|
464
|
+
total_cost: totalCost > 0 ? totalCost : undefined,
|
|
465
|
+
audio_chunks_sent: audioChunkCount,
|
|
466
|
+
audio_bytes_sent: totalAudioBytes,
|
|
467
|
+
session_id: session?.sessionId,
|
|
468
|
+
model,
|
|
469
|
+
provider: providerId,
|
|
470
|
+
});
|
|
322
471
|
yield {
|
|
323
472
|
type: 'live_end',
|
|
324
473
|
timestamp: new Date().toISOString(),
|
|
325
|
-
reason: 'completed',
|
|
474
|
+
reason: turnStatus === 'completed' ? 'completed' : 'error',
|
|
326
475
|
};
|
|
327
476
|
}
|
|
328
477
|
}
|
|
@@ -334,6 +483,10 @@ export async function ensembleLiveText(agent, options) {
|
|
|
334
483
|
const sessionGenerator = ensembleLive(config, agent, options);
|
|
335
484
|
const eventQueue = [];
|
|
336
485
|
let eventPromiseResolve = null;
|
|
486
|
+
const firstEvent = await sessionGenerator.next();
|
|
487
|
+
if (!firstEvent.done && firstEvent.value) {
|
|
488
|
+
eventQueue.push(firstEvent.value);
|
|
489
|
+
}
|
|
337
490
|
(async () => {
|
|
338
491
|
for await (const event of sessionGenerator) {
|
|
339
492
|
if (event.type === 'live_start') {
|