@juspay/neurolink 13.4.10 → 13.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -3
- package/README.md +3 -2
- package/dist/adapters/video/videoAnalyzer.d.ts +1 -0
- package/dist/adapters/video/videoAnalyzer.js +4 -0
- package/dist/autoresearch/index.d.ts +1 -0
- package/dist/autoresearch/index.js +4 -0
- package/dist/autoresearch/worker.js +4 -0
- package/dist/browser/neurolink.min.js +466 -466
- package/dist/cli/commands/decide.js +25 -5
- package/dist/cli/commands/observability.js +11 -4
- package/dist/cli/commands/telemetry.js +36 -6
- package/dist/cli/utils/formatters.d.ts +17 -0
- package/dist/cli/utils/formatters.js +39 -0
- package/dist/constants/langfuse.d.ts +17 -1
- package/dist/constants/langfuse.js +23 -1
- package/dist/constants/telemetry.d.ts +6 -0
- package/dist/constants/telemetry.js +6 -0
- package/dist/core/baseProvider.js +2 -0
- package/dist/core/geminiLoopAdapter.js +19 -0
- package/dist/core/loopEngine.js +10 -7
- package/dist/core/modules/StreamHandler.js +5 -4
- package/dist/core/modules/TelemetryHandler.d.ts +5 -1
- package/dist/core/modules/TelemetryHandler.js +16 -0
- package/dist/core/modules/ToolsManager.js +3 -0
- package/dist/core/nativeGenerateLoop.js +1 -1
- package/dist/evaluation/index.d.ts +1 -0
- package/dist/evaluation/index.js +4 -0
- package/dist/evaluation/ragasEvaluator.js +1 -0
- package/dist/factories/providerDescriptors.d.ts +4 -3
- package/dist/factories/providerDescriptors.js +6 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/mcp/externalServerManager.js +6 -2
- package/dist/mcp/toolRegistry.js +3 -0
- package/dist/neurolink.d.ts +20 -10
- package/dist/neurolink.js +134 -65
- package/dist/observability/FEATURE-STATUS.md +0 -4
- package/dist/observability/exporterRegistry.d.ts +6 -1
- package/dist/observability/exporterRegistry.js +18 -0
- package/dist/observability/exporterSpanProcessor.d.ts +20 -6
- package/dist/observability/exporterSpanProcessor.js +166 -34
- package/dist/observability/exporters/baseExporter.d.ts +22 -3
- package/dist/observability/exporters/baseExporter.js +87 -5
- package/dist/observability/exporters/langfuseExporter.d.ts +29 -3
- package/dist/observability/exporters/langfuseExporter.js +251 -41
- package/dist/observability/langfuseMasking.d.ts +1 -1
- package/dist/observability/langfuseMasking.js +1 -1
- package/dist/observability/otlpEncoding.d.ts +18 -5
- package/dist/observability/otlpEncoding.js +69 -12
- package/dist/observability/utils/spanSerializer.d.ts +3 -2
- package/dist/observability/utils/spanSerializer.js +5 -3
- package/dist/processors/base/BaseFileProcessor.d.ts +1 -0
- package/dist/processors/base/BaseFileProcessor.js +4 -0
- package/dist/processors/index.d.ts +1 -0
- package/dist/processors/index.js +4 -0
- package/dist/processors/registry/ProcessorRegistry.d.ts +1 -0
- package/dist/processors/registry/ProcessorRegistry.js +5 -0
- package/dist/providers/amazonBedrock/client.js +7 -5
- package/dist/providers/amazonSagemaker.js +32 -12
- package/dist/providers/anthropic/client.js +1 -0
- package/dist/providers/anthropic/loopAdapter.js +7 -3
- package/dist/providers/catalog/provider-catalog.schema.json +64 -3
- package/dist/providers/catalog/schema.js +81 -1
- package/dist/providers/catalog/xor.json +30 -10
- package/dist/providers/configuredDecision.d.ts +8 -1
- package/dist/providers/configuredDecision.js +22 -0
- package/dist/providers/decide/dialects/openaiDecisions.js +8 -0
- package/dist/providers/decide/dialects/systemOne.js +28 -2
- package/dist/providers/decide/guards.d.ts +6 -0
- package/dist/providers/decide/guards.js +22 -0
- package/dist/providers/googleAiStudio/client.js +2 -0
- package/dist/providers/googleVertex/client.d.ts +6 -5
- package/dist/providers/googleVertex/client.js +46 -19
- package/dist/providers/openaiChatCompletionsBase.js +4 -4
- package/dist/providers/systemOneDecision.d.ts +30 -7
- package/dist/providers/systemOneDecision.js +71 -14
- package/dist/proxy/codexUsage.js +59 -2
- package/dist/proxy/proxyTracer.d.ts +25 -1
- package/dist/proxy/proxyTracer.js +146 -22
- package/dist/proxy/proxyTranslationEngine.js +4 -2
- package/dist/proxy/sseInterceptor.js +6 -4
- package/dist/rag/chunkers/BaseChunker.js +1 -0
- package/dist/rag/index.d.ts +1 -0
- package/dist/rag/index.js +4 -0
- package/dist/rag/ragIntegration.js +1 -0
- package/dist/rag/reranker/reranker.js +2 -0
- package/dist/rag/retrieval/vectorQueryTool.js +1 -0
- package/dist/server/routes/codexProxyRoutes.js +3 -1
- package/dist/server/utils/validation.d.ts +3 -3
- package/dist/services/server/ai/observability/instrumentation.d.ts +8 -7
- package/dist/services/server/ai/observability/instrumentation.js +217 -47
- package/dist/services/server/ai/observability/langfuseSetup.d.ts +19 -1
- package/dist/services/server/ai/observability/langfuseSetup.js +62 -4
- package/dist/services/server/ai/observability/otlpSetup.d.ts +12 -2
- package/dist/services/server/ai/observability/otlpSetup.js +55 -4
- package/dist/telemetry/aiMetrics.d.ts +56 -0
- package/dist/telemetry/aiMetrics.js +229 -0
- package/dist/telemetry/attributes.d.ts +24 -1
- package/dist/telemetry/attributes.js +94 -9
- package/dist/telemetry/chatMessages.d.ts +6 -0
- package/dist/telemetry/chatMessages.js +37 -0
- package/dist/telemetry/index.d.ts +1 -1
- package/dist/telemetry/index.js +1 -1
- package/dist/telemetry/observation.d.ts +5 -5
- package/dist/telemetry/observation.js +128 -39
- package/dist/telemetry/spanSanitizer.d.ts +2 -2
- package/dist/telemetry/spanSanitizer.js +76 -8
- package/dist/telemetry/telemetryStarter.d.ts +16 -0
- package/dist/telemetry/telemetryStarter.js +28 -0
- package/dist/telemetry/traceContext.d.ts +3 -3
- package/dist/telemetry/traceContext.js +3 -3
- package/dist/telemetry/tracers.d.ts +13 -1
- package/dist/telemetry/tracers.js +49 -10
- package/dist/telemetry/withSpan.d.ts +19 -3
- package/dist/telemetry/withSpan.js +149 -20
- package/dist/types/cli.d.ts +2 -0
- package/dist/types/decision.d.ts +117 -10
- package/dist/types/exporter.d.ts +98 -3
- package/dist/types/livekit.d.ts +16 -1
- package/dist/types/loopEngine.d.ts +16 -1
- package/dist/types/observability.d.ts +38 -5
- package/dist/types/providerCatalog.d.ts +12 -1
- package/dist/types/providers.d.ts +2 -0
- package/dist/types/proxy.d.ts +2 -0
- package/dist/types/span.d.ts +26 -0
- package/dist/utils/decisionAnswers.d.ts +7 -1
- package/dist/utils/decisionAnswers.js +43 -2
- package/dist/utils/decisionLimits.d.ts +13 -1
- package/dist/utils/decisionLimits.js +52 -1
- package/dist/utils/decisionMedia.d.ts +4 -2
- package/dist/utils/decisionMedia.js +216 -14
- package/dist/utils/timeout.js +4 -2
- package/dist/voice/livekit/realtimeMcpTools.js +11 -2
- package/dist/voice/livekit/realtimeMemory.d.ts +20 -2
- package/dist/voice/livekit/realtimeMemory.js +139 -46
- package/dist/voice/livekit/realtimeVoiceAgent.js +20 -8
- package/dist/voice/livekit/turnRecorder.js +22 -9
- package/dist/workflow/core/ensembleExecutor.js +2 -0
- package/dist/workflow/core/judgeScorer.js +1 -0
- package/dist/workflow/core/responseConditioner.js +1 -0
- package/dist/workflow/core/workflowRunner.js +6 -1
- package/dist/workflow/index.d.ts +1 -0
- package/dist/workflow/index.js +4 -0
- package/docs-site/static/search-index.json +56 -48
- package/package.json +21 -2
|
@@ -7,11 +7,13 @@ const MAX_ECHOED_PATH_CHARS = 200;
|
|
|
7
7
|
const DATA_URL = {
|
|
8
8
|
image: /^data:image\/[a-z0-9.+-]+;base64,/i,
|
|
9
9
|
video: /^data:video\/[a-z0-9.+-]+;base64,/i,
|
|
10
|
+
audio: /^data:audio\/[a-z0-9.+-]+;base64,/i,
|
|
10
11
|
};
|
|
11
12
|
const WEBM_HEADER = [0x1a, 0x45, 0xdf, 0xa3];
|
|
12
13
|
const NOT_RECOGNISED = {
|
|
13
14
|
image: "is not a recognised image format",
|
|
14
15
|
video: "is not a recognised video format (MP4, MOV or WebM)",
|
|
16
|
+
audio: "is not a recognised audio format (WAV, FLAC, Ogg or MP3)",
|
|
15
17
|
};
|
|
16
18
|
/**
|
|
17
19
|
* MP4 and MOV share the ISO-BMFF box. An AVIF or HEIC image shares it too, and
|
|
@@ -26,6 +28,153 @@ function sniffVideoMimeType(buffer) {
|
|
|
26
28
|
? "video/webm"
|
|
27
29
|
: null;
|
|
28
30
|
}
|
|
31
|
+
const ascii = (buffer, start, end) => buffer.subarray(start, end).toString("latin1");
|
|
32
|
+
/**
|
|
33
|
+
* The audio containers a decision model is known to decode. MP4 audio (M4A,
|
|
34
|
+
* AAC) is told apart so it can be refused by name: it was measured decoding to
|
|
35
|
+
* silence, with the request still answered as if it had been heard.
|
|
36
|
+
*/
|
|
37
|
+
function sniffAudioFormat(buffer) {
|
|
38
|
+
if (ascii(buffer, 0, 4) === "RIFF" && ascii(buffer, 8, 12) === "WAVE") {
|
|
39
|
+
return "wav";
|
|
40
|
+
}
|
|
41
|
+
if (ascii(buffer, 0, 4) === "fLaC") {
|
|
42
|
+
return "flac";
|
|
43
|
+
}
|
|
44
|
+
if (ascii(buffer, 0, 4) === "OggS") {
|
|
45
|
+
return "ogg";
|
|
46
|
+
}
|
|
47
|
+
if (hasFtypBoxSignature(buffer)) {
|
|
48
|
+
return "aac";
|
|
49
|
+
}
|
|
50
|
+
if (ascii(buffer, 0, 3) === "ID3") {
|
|
51
|
+
return "mp3";
|
|
52
|
+
}
|
|
53
|
+
// An MPEG frame sync. Layer bits 00 mark an ADTS (AAC) stream instead.
|
|
54
|
+
if (buffer.length >= 2 && buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) {
|
|
55
|
+
return (buffer[1] & 0x06) === 0 ? "aac" : "mp3";
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
/** A WAV clip's length from its `fmt ` byte rate and `data` size, or undefined. */
|
|
60
|
+
function wavSeconds(buffer) {
|
|
61
|
+
let byteRate;
|
|
62
|
+
let offset = 12;
|
|
63
|
+
while (offset + 8 <= buffer.length) {
|
|
64
|
+
const id = ascii(buffer, offset, offset + 4);
|
|
65
|
+
const size = buffer.readUInt32LE(offset + 4);
|
|
66
|
+
const body = offset + 8;
|
|
67
|
+
if (id === "fmt " && body + 12 <= buffer.length) {
|
|
68
|
+
byteRate = buffer.readUInt32LE(body + 8);
|
|
69
|
+
}
|
|
70
|
+
else if (id === "data") {
|
|
71
|
+
// A streamed WAV leaves the size unset; what follows is the data.
|
|
72
|
+
const dataBytes = size === 0 || size === 0xffffffff
|
|
73
|
+
? buffer.length - body
|
|
74
|
+
: Math.min(size, buffer.length - body);
|
|
75
|
+
return byteRate ? dataBytes / byteRate : undefined;
|
|
76
|
+
}
|
|
77
|
+
offset = body + size + (size % 2);
|
|
78
|
+
}
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
/** A FLAC clip's length from its STREAMINFO block, or undefined when unset. */
|
|
82
|
+
function flacSeconds(buffer) {
|
|
83
|
+
// "fLaC", a 4-byte block header, then STREAMINFO.
|
|
84
|
+
if (buffer.length < 26 || (buffer[4] & 0x7f) !== 0) {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
const info = 8;
|
|
88
|
+
const sampleRate = (buffer[info + 10] << 12) |
|
|
89
|
+
(buffer[info + 11] << 4) |
|
|
90
|
+
(buffer[info + 12] >> 4);
|
|
91
|
+
const totalSamples = (buffer[info + 13] & 0x0f) * 2 ** 32 + buffer.readUInt32BE(info + 14);
|
|
92
|
+
return sampleRate > 0 && totalSamples > 0
|
|
93
|
+
? totalSamples / sampleRate
|
|
94
|
+
: undefined;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* An Ogg clip's length: the granule position of its last page over the stream's
|
|
98
|
+
* sample rate. Opus counts at 48 kHz from its pre-skip; Vorbis at the rate in
|
|
99
|
+
* its identification header. Undefined for any other codec, or a stream whose
|
|
100
|
+
* pages cannot be read.
|
|
101
|
+
*/
|
|
102
|
+
function oggSeconds(buffer) {
|
|
103
|
+
// The first page carries the codec's identification packet.
|
|
104
|
+
const segments = buffer[26];
|
|
105
|
+
if (buffer.length < 28 || segments === undefined) {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
const packet = 27 + segments;
|
|
109
|
+
let rate;
|
|
110
|
+
let preSkip = 0;
|
|
111
|
+
if (ascii(buffer, packet, packet + 8) === "OpusHead") {
|
|
112
|
+
rate = 48000;
|
|
113
|
+
preSkip =
|
|
114
|
+
buffer.length >= packet + 12 ? buffer.readUInt16LE(packet + 10) : 0;
|
|
115
|
+
}
|
|
116
|
+
else if (ascii(buffer, packet, packet + 7) === "\x01vorbis") {
|
|
117
|
+
rate =
|
|
118
|
+
buffer.length >= packet + 16
|
|
119
|
+
? buffer.readUInt32LE(packet + 12)
|
|
120
|
+
: undefined;
|
|
121
|
+
}
|
|
122
|
+
const last = buffer.lastIndexOf("OggS");
|
|
123
|
+
if (!rate ||
|
|
124
|
+
last < 0 ||
|
|
125
|
+
last + 14 > buffer.length ||
|
|
126
|
+
buffer[last + 4] !== 0) {
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
129
|
+
const granule = buffer.readBigInt64LE(last + 6);
|
|
130
|
+
if (granule <= 0n) {
|
|
131
|
+
return undefined;
|
|
132
|
+
}
|
|
133
|
+
return Math.max(0, Number(granule) - preSkip) / rate;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* The clip in wire form, or why it cannot be sent. A clip whose length can be
|
|
137
|
+
* read (WAV, FLAC, Ogg Opus or Vorbis) and that runs past what the model hears is
|
|
138
|
+
* refused; the model would answer from its start alone. An MP3's length is not
|
|
139
|
+
* read: it needs every frame header walked.
|
|
140
|
+
*/
|
|
141
|
+
function encodeAudio(buffer, limits, label) {
|
|
142
|
+
const format = sniffAudioFormat(buffer);
|
|
143
|
+
if (format === null) {
|
|
144
|
+
return { status: "error", message: `${label} ${NOT_RECOGNISED.audio}.` };
|
|
145
|
+
}
|
|
146
|
+
if (format === "aac") {
|
|
147
|
+
return {
|
|
148
|
+
status: "error",
|
|
149
|
+
message: `${label} is AAC or M4A audio, which the model does not decode. Convert it to WAV, FLAC, Ogg or MP3.`,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
const seconds = format === "wav"
|
|
153
|
+
? wavSeconds(buffer)
|
|
154
|
+
: format === "flac"
|
|
155
|
+
? flacSeconds(buffer)
|
|
156
|
+
: format === "ogg"
|
|
157
|
+
? oggSeconds(buffer)
|
|
158
|
+
: undefined;
|
|
159
|
+
// Every WAV the model can open has both chunks; the server answers one
|
|
160
|
+
// without them with an engine error, which reads as a fault worth retrying.
|
|
161
|
+
if (format === "wav" && seconds === undefined) {
|
|
162
|
+
return {
|
|
163
|
+
status: "error",
|
|
164
|
+
message: `${label} is a WAV file without a complete fmt and data chunk.`,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
if (seconds !== undefined && seconds > limits.maxSeconds) {
|
|
168
|
+
return {
|
|
169
|
+
status: "error",
|
|
170
|
+
message: `${label} runs ${seconds.toFixed(1)} seconds; the model hears at most ${limits.maxSeconds}. Trim it, or split it across requests.`,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
status: "ok",
|
|
175
|
+
audio: { data: buffer.toString("base64"), format },
|
|
176
|
+
};
|
|
177
|
+
}
|
|
29
178
|
function encode(buffer, kind, label) {
|
|
30
179
|
const mime = kind === "image" ? sniffImageMimeType(buffer) : sniffVideoMimeType(buffer);
|
|
31
180
|
return mime === null
|
|
@@ -73,14 +222,15 @@ async function readMediaFile(path, label, maxBytes) {
|
|
|
73
222
|
}
|
|
74
223
|
}
|
|
75
224
|
/**
|
|
76
|
-
*
|
|
225
|
+
* A source resolved for one kind of media, with every refusal the three kinds
|
|
226
|
+
* share. Nothing here echoes a `data:` URL or a remote URL: either can carry a
|
|
77
227
|
* credential or megabytes of base64, and neither belongs in an error message.
|
|
78
228
|
*/
|
|
79
|
-
async function
|
|
229
|
+
async function readMediaSource(source, kind, label, maxBytes) {
|
|
80
230
|
if (typeof source !== "string") {
|
|
81
231
|
return source.length === 0
|
|
82
232
|
? { status: "error", message: `${label} is an empty Buffer.` }
|
|
83
|
-
:
|
|
233
|
+
: { status: "buffer", buffer: source };
|
|
84
234
|
}
|
|
85
235
|
const value = source.trim();
|
|
86
236
|
if (value === "") {
|
|
@@ -94,7 +244,7 @@ async function toDataUrl(source, kind, label, maxBytes) {
|
|
|
94
244
|
}
|
|
95
245
|
if (/^data:/i.test(value)) {
|
|
96
246
|
return DATA_URL[kind].test(value)
|
|
97
|
-
? { status: "
|
|
247
|
+
? { status: "dataUrl", dataUrl: value }
|
|
98
248
|
: {
|
|
99
249
|
status: "error",
|
|
100
250
|
message: `${label} is a data: URL that is not base64 ${kind} content.`,
|
|
@@ -107,24 +257,59 @@ async function toDataUrl(source, kind, label, maxBytes) {
|
|
|
107
257
|
};
|
|
108
258
|
}
|
|
109
259
|
const file = await readMediaFile(value, label, maxBytes);
|
|
110
|
-
return file.status === "ok"
|
|
260
|
+
return file.status === "ok"
|
|
261
|
+
? { status: "buffer", buffer: file.buffer }
|
|
262
|
+
: file;
|
|
263
|
+
}
|
|
264
|
+
/** An image or video as a `data:` URL: bytes are identified first, a URL of the right kind is sent as given. */
|
|
265
|
+
async function toDataUrl(source, kind, label, maxBytes) {
|
|
266
|
+
const read = await readMediaSource(source, kind, label, maxBytes);
|
|
267
|
+
if (read.status === "buffer") {
|
|
268
|
+
return encode(read.buffer, kind, label);
|
|
269
|
+
}
|
|
270
|
+
return read.status === "dataUrl"
|
|
271
|
+
? { status: "ok", dataUrl: read.dataUrl }
|
|
272
|
+
: read;
|
|
273
|
+
}
|
|
274
|
+
/** An audio clip as bytes: a `data:audio/…` URL is decoded, since the wire takes raw base64 and the container is sniffed. */
|
|
275
|
+
async function readAudioSource(source, label, maxBytes) {
|
|
276
|
+
const read = await readMediaSource(source, "audio", label, maxBytes);
|
|
277
|
+
if (read.status === "dataUrl") {
|
|
278
|
+
const base64 = read.dataUrl.slice(read.dataUrl.indexOf(",") + 1);
|
|
279
|
+
return { status: "ok", buffer: Buffer.from(base64, "base64") };
|
|
280
|
+
}
|
|
281
|
+
return read.status === "buffer"
|
|
282
|
+
? { status: "ok", buffer: read.buffer }
|
|
283
|
+
: read;
|
|
111
284
|
}
|
|
112
285
|
/**
|
|
113
|
-
* Turn the caller's images and video into `data:` URLs,
|
|
114
|
-
* Runs before any request, so every
|
|
286
|
+
* Turn the caller's images and video into `data:` URLs, and an audio clip into
|
|
287
|
+
* `{ data, format }`, or say why not. Runs before any request, so every
|
|
288
|
+
* refusal costs nothing and none is retried.
|
|
115
289
|
*/
|
|
116
290
|
export async function prepareDecisionMedia(request, limits, vendor, alternatives) {
|
|
117
291
|
const images = request.images ?? [];
|
|
118
|
-
if (images.length === 0 &&
|
|
292
|
+
if (images.length === 0 &&
|
|
293
|
+
request.video === undefined &&
|
|
294
|
+
request.audio === undefined) {
|
|
119
295
|
return { status: "prepared", media: undefined };
|
|
120
296
|
}
|
|
297
|
+
const use = alternatives.length > 0
|
|
298
|
+
? ` Use a decision provider that does: ${alternatives.join(", ")}.`
|
|
299
|
+
: "";
|
|
121
300
|
if (!limits) {
|
|
122
|
-
const
|
|
123
|
-
?
|
|
124
|
-
: "";
|
|
301
|
+
const what = images.length === 0 && request.video === undefined
|
|
302
|
+
? "audio"
|
|
303
|
+
: "images or video";
|
|
125
304
|
return {
|
|
126
305
|
status: "refused",
|
|
127
|
-
message: `${vendor} does not accept
|
|
306
|
+
message: `${vendor} does not accept ${what}.${use}`,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
if (request.audio !== undefined && limits.audio === undefined) {
|
|
310
|
+
return {
|
|
311
|
+
status: "refused",
|
|
312
|
+
message: `${vendor} does not accept audio.${use}`,
|
|
128
313
|
};
|
|
129
314
|
}
|
|
130
315
|
if (images.length > limits.maxImages) {
|
|
@@ -152,9 +337,26 @@ export async function prepareDecisionMedia(request, limits, vendor, alternatives
|
|
|
152
337
|
}
|
|
153
338
|
video = converted.dataUrl;
|
|
154
339
|
}
|
|
155
|
-
|
|
340
|
+
let audio;
|
|
341
|
+
if (request.audio !== undefined && limits.audio !== undefined) {
|
|
342
|
+
const read = await readAudioSource(request.audio, "The audio", limits.maxRequestBytes);
|
|
343
|
+
if (read.status === "error") {
|
|
344
|
+
return { status: "refused", message: read.message };
|
|
345
|
+
}
|
|
346
|
+
const encoded = encodeAudio(read.buffer, limits.audio, "The audio");
|
|
347
|
+
if (encoded.status === "error") {
|
|
348
|
+
return { status: "refused", message: encoded.message };
|
|
349
|
+
}
|
|
350
|
+
audio = encoded.audio;
|
|
351
|
+
}
|
|
352
|
+
const bytes = prepared.reduce((sum, url) => sum + url.length, (video?.length ?? 0) + (audio?.data.length ?? 0));
|
|
156
353
|
return {
|
|
157
354
|
status: "prepared",
|
|
158
|
-
media: {
|
|
355
|
+
media: {
|
|
356
|
+
images: prepared,
|
|
357
|
+
...(video ? { video } : {}),
|
|
358
|
+
...(audio ? { audio } : {}),
|
|
359
|
+
bytes,
|
|
360
|
+
},
|
|
159
361
|
};
|
|
160
362
|
}
|
package/dist/utils/timeout.js
CHANGED
|
@@ -352,8 +352,10 @@ export function createTimeoutController(timeout, provider, operation) {
|
|
|
352
352
|
// from here — the setTimeout callback runs in the async context captured
|
|
353
353
|
// at schedule time, which is BEFORE the AI SDK span exists. Instead we
|
|
354
354
|
// rely on the AI SDK propagating the TimeoutError through its recordSpan
|
|
355
|
-
// wrapper, which sets span.status = ERROR + message.
|
|
356
|
-
//
|
|
355
|
+
// wrapper, which sets span.status = ERROR + message. NeuroLink's own
|
|
356
|
+
// spans the error passes through (withGeneration, withSpan) record it
|
|
357
|
+
// in markFailure as level=ERROR + status_message, and
|
|
358
|
+
// ExporterSpanProcessor exports them.
|
|
357
359
|
controller.abort(new TimeoutError(`${provider} ${operation} operation timed out after ${timeout}`, timeoutMs, provider, operation));
|
|
358
360
|
};
|
|
359
361
|
let timer = setTimeout(fire, timeoutMs);
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* See docs/features/livekit-voice-agent.md.
|
|
16
16
|
*/
|
|
17
|
+
import { randomUUID } from "node:crypto";
|
|
17
18
|
import { z } from "zod";
|
|
18
19
|
import { logger } from "../../utils/logger.js";
|
|
19
20
|
import { findSchemaIssue, sanitizeToolParameters } from "./schemaSanitizer.js";
|
|
@@ -159,7 +160,10 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
159
160
|
args: args ?? {},
|
|
160
161
|
});
|
|
161
162
|
publishEvent("tool-start", { name: mcpTool.name });
|
|
162
|
-
|
|
163
|
+
// Concurrent calls of one tool can end in any order; the id lets the
|
|
164
|
+
// turn record give each its own result.
|
|
165
|
+
const callId = randomUUID();
|
|
166
|
+
observe(() => onToolStart?.({ name: mcpTool.name, args: args ?? {}, callId }));
|
|
163
167
|
const startedAt = Date.now();
|
|
164
168
|
try {
|
|
165
169
|
// Third argument is RequestOptions; the SDK aborts the in-flight
|
|
@@ -182,6 +186,7 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
182
186
|
publishEvent("tool-result", { name: mcpTool.name, result });
|
|
183
187
|
observe(() => onToolEnd?.({
|
|
184
188
|
name: mcpTool.name,
|
|
189
|
+
callId,
|
|
185
190
|
...(result.isError === true
|
|
186
191
|
? { error: text }
|
|
187
192
|
: { result: text }),
|
|
@@ -195,7 +200,11 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
195
200
|
error: error instanceof Error ? error.message : String(error),
|
|
196
201
|
});
|
|
197
202
|
publishEvent("tool-result", { name: mcpTool.name, result: null });
|
|
198
|
-
observe(() => onToolEnd?.({
|
|
203
|
+
observe(() => onToolEnd?.({
|
|
204
|
+
name: mcpTool.name,
|
|
205
|
+
callId,
|
|
206
|
+
error: String(error),
|
|
207
|
+
}));
|
|
199
208
|
return `Tool ${mcpTool.name} failed: ${String(error)}`;
|
|
200
209
|
}
|
|
201
210
|
},
|
|
@@ -12,12 +12,30 @@
|
|
|
12
12
|
* The shutdown flush is registered here, before the job lifecycle installs its
|
|
13
13
|
* process reaper. LiveKit starts every shutdown callback in registration order
|
|
14
14
|
* and the reaper exits the process about a second later, so the flush gets a
|
|
15
|
-
* budget below that second and starts first.
|
|
15
|
+
* budget below that second and starts first. The same budget bounds delivery of
|
|
16
|
+
* the call's last traces, which go out while the last writes finish.
|
|
17
|
+
*
|
|
18
|
+
* Each turn is traced as it closes, never behind the store's one-at-a-time
|
|
19
|
+
* writes, so a slow or stuck store cannot delay or lose a trace.
|
|
20
|
+
*
|
|
21
|
+
* Tracing does not depend on memory. A call with no memory still gets a
|
|
22
|
+
* recorder that only traces its turns (`setUpRealtimeCallTracing`).
|
|
16
23
|
*
|
|
17
24
|
* See docs/features/livekit-voice-agent.md.
|
|
18
25
|
*/
|
|
19
26
|
import type { JobContext } from "@livekit/agents";
|
|
20
|
-
import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig } from "../../types/index.js";
|
|
27
|
+
import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig, VoiceTurnRecorder } from "../../types/index.js";
|
|
28
|
+
/**
|
|
29
|
+
* Prepare a recorder that only traces: for a call with no usable memory. Returns
|
|
30
|
+
* `undefined` when tracing is off. Without a verified binding there is no user,
|
|
31
|
+
* so the room is the session.
|
|
32
|
+
*/
|
|
33
|
+
export declare function setUpRealtimeCallTracing(ctx: Pick<JobContext, "room" | "addShutdownCallback">, options?: {
|
|
34
|
+
model?: string;
|
|
35
|
+
backend?: RealtimeVoiceBackend;
|
|
36
|
+
settleMs?: number;
|
|
37
|
+
flushBudgetMs?: number;
|
|
38
|
+
}): VoiceTurnRecorder | undefined;
|
|
21
39
|
/**
|
|
22
40
|
* Verify the call's binding and prepare its recorder; `undefined` means "no
|
|
23
41
|
* memory for this call". `call` names the model and the realtime backend the
|
|
@@ -12,13 +12,21 @@
|
|
|
12
12
|
* The shutdown flush is registered here, before the job lifecycle installs its
|
|
13
13
|
* process reaper. LiveKit starts every shutdown callback in registration order
|
|
14
14
|
* and the reaper exits the process about a second later, so the flush gets a
|
|
15
|
-
* budget below that second and starts first.
|
|
15
|
+
* budget below that second and starts first. The same budget bounds delivery of
|
|
16
|
+
* the call's last traces, which go out while the last writes finish.
|
|
17
|
+
*
|
|
18
|
+
* Each turn is traced as it closes, never behind the store's one-at-a-time
|
|
19
|
+
* writes, so a slow or stuck store cannot delay or lose a trace.
|
|
20
|
+
*
|
|
21
|
+
* Tracing does not depend on memory. A call with no memory still gets a
|
|
22
|
+
* recorder that only traces its turns (`setUpRealtimeCallTracing`).
|
|
16
23
|
*
|
|
17
24
|
* See docs/features/livekit-voice-agent.md.
|
|
18
25
|
*/
|
|
19
26
|
import { context, trace } from "@opentelemetry/api";
|
|
20
27
|
import { AIProviderName } from "../../constants/enums.js";
|
|
21
|
-
import { setLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
|
|
28
|
+
import { flushOpenTelemetry, isOpenTelemetryInitialized, setLangfuseContext, } from "../../services/server/ai/observability/instrumentation.js";
|
|
29
|
+
import { isLangfuseEnabled } from "../../services/server/ai/observability/langfuseSetup.js";
|
|
22
30
|
import { observe, observeTrace } from "../../telemetry/observation.js";
|
|
23
31
|
import { tracers } from "../../telemetry/tracers.js";
|
|
24
32
|
import { logger } from "../../utils/logger.js";
|
|
@@ -31,7 +39,7 @@ const TRACE_PROVIDER = {
|
|
|
31
39
|
xai: AIProviderName.XAI,
|
|
32
40
|
};
|
|
33
41
|
function traceTurn(turn, call) {
|
|
34
|
-
const { sessionId, userId, roomName, model, provider } = call;
|
|
42
|
+
const { sessionId, userId, roomName, model, provider = AIProviderName.VERTEX, } = call;
|
|
35
43
|
try {
|
|
36
44
|
setLangfuseContext({
|
|
37
45
|
sessionId,
|
|
@@ -45,39 +53,62 @@ function traceTurn(turn, call) {
|
|
|
45
53
|
const root = tracers.sdk.startSpan("realtime-turn", {
|
|
46
54
|
startTime: turn.startTime,
|
|
47
55
|
});
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
output: turn.aiResponse,
|
|
56
|
-
});
|
|
57
|
-
const inRoot = trace.setSpan(context.active(), root);
|
|
58
|
-
const generation = tracers.generation.startSpan("realtime", { startTime: turn.startTime }, inRoot);
|
|
59
|
-
observe(generation, "generation", {
|
|
60
|
-
...(model && { model }),
|
|
61
|
-
input: [{ role: "user", content: turn.userMessage }],
|
|
62
|
-
output: [{ role: "assistant", content: turn.aiResponse }],
|
|
63
|
-
level,
|
|
64
|
-
metadata: { provider },
|
|
65
|
-
});
|
|
66
|
-
generation.end(endTime);
|
|
67
|
-
const events = turn.events ?? [];
|
|
68
|
-
for (const start of events.filter((e) => e.type === "tool:start")) {
|
|
69
|
-
const end = events.find((e) => e.type === "tool:end" &&
|
|
70
|
-
e.tool === start.tool &&
|
|
71
|
-
e.seq > start.seq);
|
|
72
|
-
const tool = tracers.sdk.startSpan(String(start.tool), { startTime: start.timestamp }, inRoot);
|
|
73
|
-
observe(tool, "tool", {
|
|
74
|
-
input: start.args,
|
|
75
|
-
output: end?.result ?? end?.error,
|
|
76
|
-
level: end?.success === false ? "ERROR" : undefined,
|
|
56
|
+
// The root and each child end on every path, so a failure while
|
|
57
|
+
// describing one tool cannot leave the turn open and unexported.
|
|
58
|
+
try {
|
|
59
|
+
observe(root, "agent", {
|
|
60
|
+
input: turn.userMessage,
|
|
61
|
+
output: turn.aiResponse,
|
|
62
|
+
level,
|
|
77
63
|
});
|
|
78
|
-
|
|
64
|
+
observeTrace(root, {
|
|
65
|
+
input: turn.userMessage,
|
|
66
|
+
output: turn.aiResponse,
|
|
67
|
+
});
|
|
68
|
+
const inRoot = trace.setSpan(context.active(), root);
|
|
69
|
+
const generation = tracers.generation.startSpan("realtime", { startTime: turn.startTime }, inRoot);
|
|
70
|
+
try {
|
|
71
|
+
observe(generation, "generation", {
|
|
72
|
+
...(model && { model }),
|
|
73
|
+
input: [{ role: "user", content: turn.userMessage }],
|
|
74
|
+
output: [{ role: "assistant", content: turn.aiResponse }],
|
|
75
|
+
level,
|
|
76
|
+
metadata: { provider },
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
generation.end(endTime);
|
|
81
|
+
}
|
|
82
|
+
const events = turn.events ?? [];
|
|
83
|
+
const consumed = new Set();
|
|
84
|
+
for (const start of events.filter((e) => e.type === "tool:start")) {
|
|
85
|
+
// A call id pairs overlapping calls exactly; without one, each end
|
|
86
|
+
// is spent on the earliest unpaired start of the same tool.
|
|
87
|
+
const end = events.find((e) => e.type === "tool:end" &&
|
|
88
|
+
e.seq > start.seq &&
|
|
89
|
+
!consumed.has(e.seq) &&
|
|
90
|
+
(start.callId !== undefined
|
|
91
|
+
? e.callId === start.callId
|
|
92
|
+
: e.tool === start.tool));
|
|
93
|
+
if (end) {
|
|
94
|
+
consumed.add(end.seq);
|
|
95
|
+
}
|
|
96
|
+
const tool = tracers.sdk.startSpan(String(start.tool), { startTime: start.timestamp }, inRoot);
|
|
97
|
+
try {
|
|
98
|
+
observe(tool, "tool", {
|
|
99
|
+
input: start.args,
|
|
100
|
+
output: end?.result ?? end?.error,
|
|
101
|
+
level: end?.success === false ? "ERROR" : undefined,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
tool.end(Math.min(end?.timestamp ?? Infinity, endTime.getTime()));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
root.end(endTime);
|
|
79
111
|
}
|
|
80
|
-
root.end(endTime);
|
|
81
112
|
});
|
|
82
113
|
}
|
|
83
114
|
catch (error) {
|
|
@@ -86,6 +117,71 @@ function traceTurn(turn, call) {
|
|
|
86
117
|
});
|
|
87
118
|
}
|
|
88
119
|
}
|
|
120
|
+
/** A tracer that spends one trace per turn id. */
|
|
121
|
+
function createTurnTracer(call) {
|
|
122
|
+
const tracedTurns = new Set();
|
|
123
|
+
return (turn) => {
|
|
124
|
+
if (!tracedTurns.has(turn.turnId)) {
|
|
125
|
+
tracedTurns.add(turn.turnId);
|
|
126
|
+
traceTurn(turn, call);
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/** Deliver buffered traces, giving up after `budgetMs`: the worker is reaped about a second after shutdown starts. */
|
|
131
|
+
async function flushTraces(budgetMs) {
|
|
132
|
+
let timer;
|
|
133
|
+
try {
|
|
134
|
+
await Promise.race([
|
|
135
|
+
flushOpenTelemetry(),
|
|
136
|
+
new Promise((resolve) => {
|
|
137
|
+
timer = setTimeout(resolve, Math.max(0, budgetMs));
|
|
138
|
+
}),
|
|
139
|
+
]);
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
logger.debug("realtime.memory.traceFlushFailed", {
|
|
143
|
+
error: error instanceof Error ? error.message : String(error),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
clearTimeout(timer);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/** Shutdown: close the open exchange, then deliver every trace while its writes finish, all within one budget. */
|
|
151
|
+
async function flushCall(recorder, budgetMs) {
|
|
152
|
+
// flush() closes, and so traces, the open exchange before it first waits.
|
|
153
|
+
const [stats] = await Promise.all([
|
|
154
|
+
recorder.flush(budgetMs),
|
|
155
|
+
flushTraces(budgetMs),
|
|
156
|
+
]);
|
|
157
|
+
return stats;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Prepare a recorder that only traces: for a call with no usable memory. Returns
|
|
161
|
+
* `undefined` when tracing is off. Without a verified binding there is no user,
|
|
162
|
+
* so the room is the session.
|
|
163
|
+
*/
|
|
164
|
+
export function setUpRealtimeCallTracing(ctx, options = {}) {
|
|
165
|
+
if (!isLangfuseEnabled() && !isOpenTelemetryInitialized()) {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
const trace = createTurnTracer({
|
|
169
|
+
sessionId: ctx.room.name,
|
|
170
|
+
roomName: ctx.room.name,
|
|
171
|
+
model: options.model,
|
|
172
|
+
provider: TRACE_PROVIDER[options.backend ?? "gemini"],
|
|
173
|
+
});
|
|
174
|
+
const recorder = createVoiceTurnRecorder({
|
|
175
|
+
...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
|
|
176
|
+
onClosed: trace,
|
|
177
|
+
write: () => Promise.resolve(),
|
|
178
|
+
});
|
|
179
|
+
ctx.addShutdownCallback(async () => {
|
|
180
|
+
const stats = await flushCall(recorder, options.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
|
|
181
|
+
logger.info("realtime.trace.flushed", { ...stats });
|
|
182
|
+
});
|
|
183
|
+
return recorder;
|
|
184
|
+
}
|
|
89
185
|
/**
|
|
90
186
|
* Verify the call's binding and prepare its recorder; `undefined` means "no
|
|
91
187
|
* memory for this call". `call` names the model and the realtime backend the
|
|
@@ -122,20 +218,17 @@ export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
|
|
|
122
218
|
return undefined;
|
|
123
219
|
}
|
|
124
220
|
let publish;
|
|
125
|
-
const
|
|
221
|
+
const trace = createTurnTracer({
|
|
222
|
+
sessionId,
|
|
223
|
+
userId,
|
|
224
|
+
roomName: ctx.room.name,
|
|
225
|
+
model,
|
|
226
|
+
provider: TRACE_PROVIDER[backend],
|
|
227
|
+
});
|
|
126
228
|
const recorder = createVoiceTurnRecorder({
|
|
127
229
|
...(config.settleMs !== undefined ? { settleMs: config.settleMs } : {}),
|
|
230
|
+
onClosed: trace,
|
|
128
231
|
write: async (turn) => {
|
|
129
|
-
if (!tracedTurns.has(turn.turnId)) {
|
|
130
|
-
tracedTurns.add(turn.turnId);
|
|
131
|
-
traceTurn(turn, {
|
|
132
|
-
sessionId,
|
|
133
|
-
userId,
|
|
134
|
-
roomName: ctx.room.name,
|
|
135
|
-
model,
|
|
136
|
-
provider: TRACE_PROVIDER[backend],
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
232
|
const result = await neurolink.recordConversationTurn({
|
|
140
233
|
sessionId,
|
|
141
234
|
userId,
|
|
@@ -160,7 +253,7 @@ export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
|
|
|
160
253
|
},
|
|
161
254
|
});
|
|
162
255
|
ctx.addShutdownCallback(async () => {
|
|
163
|
-
const stats = await recorder
|
|
256
|
+
const stats = await flushCall(recorder, config.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
|
|
164
257
|
logger.info("realtime.memory.flushed", { ...stats });
|
|
165
258
|
});
|
|
166
259
|
let seed = [];
|