@juspay/neurolink 13.4.10 → 13.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/CHANGELOG.md +2 -3
  2. package/README.md +3 -2
  3. package/dist/adapters/video/videoAnalyzer.d.ts +1 -0
  4. package/dist/adapters/video/videoAnalyzer.js +4 -0
  5. package/dist/autoresearch/index.d.ts +1 -0
  6. package/dist/autoresearch/index.js +4 -0
  7. package/dist/autoresearch/worker.js +4 -0
  8. package/dist/browser/neurolink.min.js +466 -466
  9. package/dist/cli/commands/decide.js +25 -5
  10. package/dist/cli/commands/observability.js +11 -4
  11. package/dist/cli/commands/telemetry.js +36 -6
  12. package/dist/cli/utils/formatters.d.ts +17 -0
  13. package/dist/cli/utils/formatters.js +39 -0
  14. package/dist/constants/langfuse.d.ts +17 -1
  15. package/dist/constants/langfuse.js +23 -1
  16. package/dist/constants/telemetry.d.ts +6 -0
  17. package/dist/constants/telemetry.js +6 -0
  18. package/dist/core/baseProvider.js +2 -0
  19. package/dist/core/geminiLoopAdapter.js +19 -0
  20. package/dist/core/loopEngine.js +10 -7
  21. package/dist/core/modules/StreamHandler.js +5 -4
  22. package/dist/core/modules/TelemetryHandler.d.ts +5 -1
  23. package/dist/core/modules/TelemetryHandler.js +16 -0
  24. package/dist/core/modules/ToolsManager.js +3 -0
  25. package/dist/core/nativeGenerateLoop.js +1 -1
  26. package/dist/evaluation/index.d.ts +1 -0
  27. package/dist/evaluation/index.js +4 -0
  28. package/dist/evaluation/ragasEvaluator.js +1 -0
  29. package/dist/factories/providerDescriptors.d.ts +4 -3
  30. package/dist/factories/providerDescriptors.js +6 -4
  31. package/dist/index.d.ts +1 -1
  32. package/dist/index.js +1 -1
  33. package/dist/mcp/externalServerManager.js +6 -2
  34. package/dist/mcp/toolRegistry.js +3 -0
  35. package/dist/neurolink.d.ts +20 -10
  36. package/dist/neurolink.js +134 -65
  37. package/dist/observability/FEATURE-STATUS.md +0 -4
  38. package/dist/observability/exporterRegistry.d.ts +6 -1
  39. package/dist/observability/exporterRegistry.js +18 -0
  40. package/dist/observability/exporterSpanProcessor.d.ts +20 -6
  41. package/dist/observability/exporterSpanProcessor.js +166 -34
  42. package/dist/observability/exporters/baseExporter.d.ts +22 -3
  43. package/dist/observability/exporters/baseExporter.js +87 -5
  44. package/dist/observability/exporters/langfuseExporter.d.ts +29 -3
  45. package/dist/observability/exporters/langfuseExporter.js +251 -41
  46. package/dist/observability/langfuseMasking.d.ts +1 -1
  47. package/dist/observability/langfuseMasking.js +1 -1
  48. package/dist/observability/otlpEncoding.d.ts +18 -5
  49. package/dist/observability/otlpEncoding.js +69 -12
  50. package/dist/observability/utils/spanSerializer.d.ts +3 -2
  51. package/dist/observability/utils/spanSerializer.js +5 -3
  52. package/dist/processors/base/BaseFileProcessor.d.ts +1 -0
  53. package/dist/processors/base/BaseFileProcessor.js +4 -0
  54. package/dist/processors/index.d.ts +1 -0
  55. package/dist/processors/index.js +4 -0
  56. package/dist/processors/registry/ProcessorRegistry.d.ts +1 -0
  57. package/dist/processors/registry/ProcessorRegistry.js +5 -0
  58. package/dist/providers/amazonBedrock/client.js +7 -5
  59. package/dist/providers/amazonSagemaker.js +32 -12
  60. package/dist/providers/anthropic/client.js +1 -0
  61. package/dist/providers/anthropic/loopAdapter.js +7 -3
  62. package/dist/providers/catalog/provider-catalog.schema.json +64 -3
  63. package/dist/providers/catalog/schema.js +81 -1
  64. package/dist/providers/catalog/xor.json +30 -10
  65. package/dist/providers/configuredDecision.d.ts +8 -1
  66. package/dist/providers/configuredDecision.js +22 -0
  67. package/dist/providers/decide/dialects/openaiDecisions.js +8 -0
  68. package/dist/providers/decide/dialects/systemOne.js +28 -2
  69. package/dist/providers/decide/guards.d.ts +6 -0
  70. package/dist/providers/decide/guards.js +22 -0
  71. package/dist/providers/googleAiStudio/client.js +2 -0
  72. package/dist/providers/googleVertex/client.d.ts +6 -5
  73. package/dist/providers/googleVertex/client.js +46 -19
  74. package/dist/providers/openaiChatCompletionsBase.js +4 -4
  75. package/dist/providers/systemOneDecision.d.ts +30 -7
  76. package/dist/providers/systemOneDecision.js +71 -14
  77. package/dist/proxy/codexUsage.js +59 -2
  78. package/dist/proxy/proxyTracer.d.ts +25 -1
  79. package/dist/proxy/proxyTracer.js +146 -22
  80. package/dist/proxy/proxyTranslationEngine.js +4 -2
  81. package/dist/proxy/sseInterceptor.js +6 -4
  82. package/dist/rag/chunkers/BaseChunker.js +1 -0
  83. package/dist/rag/index.d.ts +1 -0
  84. package/dist/rag/index.js +4 -0
  85. package/dist/rag/ragIntegration.js +1 -0
  86. package/dist/rag/reranker/reranker.js +2 -0
  87. package/dist/rag/retrieval/vectorQueryTool.js +1 -0
  88. package/dist/server/routes/codexProxyRoutes.js +3 -1
  89. package/dist/server/utils/validation.d.ts +3 -3
  90. package/dist/services/server/ai/observability/instrumentation.d.ts +8 -7
  91. package/dist/services/server/ai/observability/instrumentation.js +217 -47
  92. package/dist/services/server/ai/observability/langfuseSetup.d.ts +19 -1
  93. package/dist/services/server/ai/observability/langfuseSetup.js +62 -4
  94. package/dist/services/server/ai/observability/otlpSetup.d.ts +12 -2
  95. package/dist/services/server/ai/observability/otlpSetup.js +55 -4
  96. package/dist/telemetry/aiMetrics.d.ts +56 -0
  97. package/dist/telemetry/aiMetrics.js +229 -0
  98. package/dist/telemetry/attributes.d.ts +24 -1
  99. package/dist/telemetry/attributes.js +94 -9
  100. package/dist/telemetry/chatMessages.d.ts +6 -0
  101. package/dist/telemetry/chatMessages.js +37 -0
  102. package/dist/telemetry/index.d.ts +1 -1
  103. package/dist/telemetry/index.js +1 -1
  104. package/dist/telemetry/observation.d.ts +5 -5
  105. package/dist/telemetry/observation.js +128 -39
  106. package/dist/telemetry/spanSanitizer.d.ts +2 -2
  107. package/dist/telemetry/spanSanitizer.js +76 -8
  108. package/dist/telemetry/telemetryStarter.d.ts +16 -0
  109. package/dist/telemetry/telemetryStarter.js +28 -0
  110. package/dist/telemetry/traceContext.d.ts +3 -3
  111. package/dist/telemetry/traceContext.js +3 -3
  112. package/dist/telemetry/tracers.d.ts +13 -1
  113. package/dist/telemetry/tracers.js +49 -10
  114. package/dist/telemetry/withSpan.d.ts +19 -3
  115. package/dist/telemetry/withSpan.js +149 -20
  116. package/dist/types/cli.d.ts +2 -0
  117. package/dist/types/decision.d.ts +117 -10
  118. package/dist/types/exporter.d.ts +98 -3
  119. package/dist/types/livekit.d.ts +16 -1
  120. package/dist/types/loopEngine.d.ts +16 -1
  121. package/dist/types/observability.d.ts +38 -5
  122. package/dist/types/providerCatalog.d.ts +12 -1
  123. package/dist/types/providers.d.ts +2 -0
  124. package/dist/types/proxy.d.ts +2 -0
  125. package/dist/types/span.d.ts +26 -0
  126. package/dist/utils/decisionAnswers.d.ts +7 -1
  127. package/dist/utils/decisionAnswers.js +43 -2
  128. package/dist/utils/decisionLimits.d.ts +13 -1
  129. package/dist/utils/decisionLimits.js +52 -1
  130. package/dist/utils/decisionMedia.d.ts +4 -2
  131. package/dist/utils/decisionMedia.js +216 -14
  132. package/dist/utils/timeout.js +4 -2
  133. package/dist/voice/livekit/realtimeMcpTools.js +11 -2
  134. package/dist/voice/livekit/realtimeMemory.d.ts +20 -2
  135. package/dist/voice/livekit/realtimeMemory.js +139 -46
  136. package/dist/voice/livekit/realtimeVoiceAgent.js +20 -8
  137. package/dist/voice/livekit/turnRecorder.js +22 -9
  138. package/dist/workflow/core/ensembleExecutor.js +2 -0
  139. package/dist/workflow/core/judgeScorer.js +1 -0
  140. package/dist/workflow/core/responseConditioner.js +1 -0
  141. package/dist/workflow/core/workflowRunner.js +6 -1
  142. package/dist/workflow/index.d.ts +1 -0
  143. package/dist/workflow/index.js +4 -0
  144. package/docs-site/static/search-index.json +56 -48
  145. package/package.json +21 -2
@@ -7,11 +7,13 @@ const MAX_ECHOED_PATH_CHARS = 200;
7
7
  const DATA_URL = {
8
8
  image: /^data:image\/[a-z0-9.+-]+;base64,/i,
9
9
  video: /^data:video\/[a-z0-9.+-]+;base64,/i,
10
+ audio: /^data:audio\/[a-z0-9.+-]+;base64,/i,
10
11
  };
11
12
  const WEBM_HEADER = [0x1a, 0x45, 0xdf, 0xa3];
12
13
  const NOT_RECOGNISED = {
13
14
  image: "is not a recognised image format",
14
15
  video: "is not a recognised video format (MP4, MOV or WebM)",
16
+ audio: "is not a recognised audio format (WAV, FLAC, Ogg or MP3)",
15
17
  };
16
18
  /**
17
19
  * MP4 and MOV share the ISO-BMFF box. An AVIF or HEIC image shares it too, and
@@ -26,6 +28,153 @@ function sniffVideoMimeType(buffer) {
26
28
  ? "video/webm"
27
29
  : null;
28
30
  }
31
+ const ascii = (buffer, start, end) => buffer.subarray(start, end).toString("latin1");
32
+ /**
33
+ * The audio containers a decision model is known to decode. MP4 audio (M4A,
34
+ * AAC) is told apart so it can be refused by name: it was measured decoding to
35
+ * silence, with the request still answered as if it had been heard.
36
+ */
37
+ function sniffAudioFormat(buffer) {
38
+ if (ascii(buffer, 0, 4) === "RIFF" && ascii(buffer, 8, 12) === "WAVE") {
39
+ return "wav";
40
+ }
41
+ if (ascii(buffer, 0, 4) === "fLaC") {
42
+ return "flac";
43
+ }
44
+ if (ascii(buffer, 0, 4) === "OggS") {
45
+ return "ogg";
46
+ }
47
+ if (hasFtypBoxSignature(buffer)) {
48
+ return "aac";
49
+ }
50
+ if (ascii(buffer, 0, 3) === "ID3") {
51
+ return "mp3";
52
+ }
53
+ // An MPEG frame sync. Layer bits 00 mark an ADTS (AAC) stream instead.
54
+ if (buffer.length >= 2 && buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) {
55
+ return (buffer[1] & 0x06) === 0 ? "aac" : "mp3";
56
+ }
57
+ return null;
58
+ }
59
+ /** A WAV clip's length from its `fmt ` byte rate and `data` size, or undefined. */
60
+ function wavSeconds(buffer) {
61
+ let byteRate;
62
+ let offset = 12;
63
+ while (offset + 8 <= buffer.length) {
64
+ const id = ascii(buffer, offset, offset + 4);
65
+ const size = buffer.readUInt32LE(offset + 4);
66
+ const body = offset + 8;
67
+ if (id === "fmt " && body + 12 <= buffer.length) {
68
+ byteRate = buffer.readUInt32LE(body + 8);
69
+ }
70
+ else if (id === "data") {
71
+ // A streamed WAV leaves the size unset; what follows is the data.
72
+ const dataBytes = size === 0 || size === 0xffffffff
73
+ ? buffer.length - body
74
+ : Math.min(size, buffer.length - body);
75
+ return byteRate ? dataBytes / byteRate : undefined;
76
+ }
77
+ offset = body + size + (size % 2);
78
+ }
79
+ return undefined;
80
+ }
81
+ /** A FLAC clip's length from its STREAMINFO block, or undefined when unset. */
82
+ function flacSeconds(buffer) {
83
+ // "fLaC", a 4-byte block header, then STREAMINFO.
84
+ if (buffer.length < 26 || (buffer[4] & 0x7f) !== 0) {
85
+ return undefined;
86
+ }
87
+ const info = 8;
88
+ const sampleRate = (buffer[info + 10] << 12) |
89
+ (buffer[info + 11] << 4) |
90
+ (buffer[info + 12] >> 4);
91
+ const totalSamples = (buffer[info + 13] & 0x0f) * 2 ** 32 + buffer.readUInt32BE(info + 14);
92
+ return sampleRate > 0 && totalSamples > 0
93
+ ? totalSamples / sampleRate
94
+ : undefined;
95
+ }
96
+ /**
97
+ * An Ogg clip's length: the granule position of its last page over the stream's
98
+ * sample rate. Opus counts at 48 kHz from its pre-skip; Vorbis at the rate in
99
+ * its identification header. Undefined for any other codec, or a stream whose
100
+ * pages cannot be read.
101
+ */
102
+ function oggSeconds(buffer) {
103
+ // The first page carries the codec's identification packet.
104
+ const segments = buffer[26];
105
+ if (buffer.length < 28 || segments === undefined) {
106
+ return undefined;
107
+ }
108
+ const packet = 27 + segments;
109
+ let rate;
110
+ let preSkip = 0;
111
+ if (ascii(buffer, packet, packet + 8) === "OpusHead") {
112
+ rate = 48000;
113
+ preSkip =
114
+ buffer.length >= packet + 12 ? buffer.readUInt16LE(packet + 10) : 0;
115
+ }
116
+ else if (ascii(buffer, packet, packet + 7) === "\x01vorbis") {
117
+ rate =
118
+ buffer.length >= packet + 16
119
+ ? buffer.readUInt32LE(packet + 12)
120
+ : undefined;
121
+ }
122
+ const last = buffer.lastIndexOf("OggS");
123
+ if (!rate ||
124
+ last < 0 ||
125
+ last + 14 > buffer.length ||
126
+ buffer[last + 4] !== 0) {
127
+ return undefined;
128
+ }
129
+ const granule = buffer.readBigInt64LE(last + 6);
130
+ if (granule <= 0n) {
131
+ return undefined;
132
+ }
133
+ return Math.max(0, Number(granule) - preSkip) / rate;
134
+ }
135
+ /**
136
+ * The clip in wire form, or why it cannot be sent. A clip whose length can be
137
+ * read (WAV, FLAC, Ogg Opus or Vorbis) and that runs past what the model hears is
138
+ * refused; the model would answer from its start alone. An MP3's length is not
139
+ * read: it needs every frame header walked.
140
+ */
141
+ function encodeAudio(buffer, limits, label) {
142
+ const format = sniffAudioFormat(buffer);
143
+ if (format === null) {
144
+ return { status: "error", message: `${label} ${NOT_RECOGNISED.audio}.` };
145
+ }
146
+ if (format === "aac") {
147
+ return {
148
+ status: "error",
149
+ message: `${label} is AAC or M4A audio, which the model does not decode. Convert it to WAV, FLAC, Ogg or MP3.`,
150
+ };
151
+ }
152
+ const seconds = format === "wav"
153
+ ? wavSeconds(buffer)
154
+ : format === "flac"
155
+ ? flacSeconds(buffer)
156
+ : format === "ogg"
157
+ ? oggSeconds(buffer)
158
+ : undefined;
159
+ // Every WAV the model can open has both chunks; the server answers one
160
+ // without them with an engine error, which reads as a fault worth retrying.
161
+ if (format === "wav" && seconds === undefined) {
162
+ return {
163
+ status: "error",
164
+ message: `${label} is a WAV file without a complete fmt and data chunk.`,
165
+ };
166
+ }
167
+ if (seconds !== undefined && seconds > limits.maxSeconds) {
168
+ return {
169
+ status: "error",
170
+ message: `${label} runs ${seconds.toFixed(1)} seconds; the model hears at most ${limits.maxSeconds}. Trim it, or split it across requests.`,
171
+ };
172
+ }
173
+ return {
174
+ status: "ok",
175
+ audio: { data: buffer.toString("base64"), format },
176
+ };
177
+ }
29
178
  function encode(buffer, kind, label) {
30
179
  const mime = kind === "image" ? sniffImageMimeType(buffer) : sniffVideoMimeType(buffer);
31
180
  return mime === null
@@ -73,14 +222,15 @@ async function readMediaFile(path, label, maxBytes) {
73
222
  }
74
223
  }
75
224
  /**
76
- * Nothing here echoes a `data:` URL or a remote URL: either can carry a
225
+ * A source resolved for one kind of media, with every refusal the three kinds
226
+ * share. Nothing here echoes a `data:` URL or a remote URL: either can carry a
77
227
  * credential or megabytes of base64, and neither belongs in an error message.
78
228
  */
79
- async function toDataUrl(source, kind, label, maxBytes) {
229
+ async function readMediaSource(source, kind, label, maxBytes) {
80
230
  if (typeof source !== "string") {
81
231
  return source.length === 0
82
232
  ? { status: "error", message: `${label} is an empty Buffer.` }
83
- : encode(source, kind, label);
233
+ : { status: "buffer", buffer: source };
84
234
  }
85
235
  const value = source.trim();
86
236
  if (value === "") {
@@ -94,7 +244,7 @@ async function toDataUrl(source, kind, label, maxBytes) {
94
244
  }
95
245
  if (/^data:/i.test(value)) {
96
246
  return DATA_URL[kind].test(value)
97
- ? { status: "ok", dataUrl: value }
247
+ ? { status: "dataUrl", dataUrl: value }
98
248
  : {
99
249
  status: "error",
100
250
  message: `${label} is a data: URL that is not base64 ${kind} content.`,
@@ -107,24 +257,59 @@ async function toDataUrl(source, kind, label, maxBytes) {
107
257
  };
108
258
  }
109
259
  const file = await readMediaFile(value, label, maxBytes);
110
- return file.status === "ok" ? encode(file.buffer, kind, label) : file;
260
+ return file.status === "ok"
261
+ ? { status: "buffer", buffer: file.buffer }
262
+ : file;
263
+ }
264
+ /** An image or video as a `data:` URL: bytes are identified first, a URL of the right kind is sent as given. */
265
+ async function toDataUrl(source, kind, label, maxBytes) {
266
+ const read = await readMediaSource(source, kind, label, maxBytes);
267
+ if (read.status === "buffer") {
268
+ return encode(read.buffer, kind, label);
269
+ }
270
+ return read.status === "dataUrl"
271
+ ? { status: "ok", dataUrl: read.dataUrl }
272
+ : read;
273
+ }
274
+ /** An audio clip as bytes: a `data:audio/…` URL is decoded, since the wire takes raw base64 and the container is sniffed. */
275
+ async function readAudioSource(source, label, maxBytes) {
276
+ const read = await readMediaSource(source, "audio", label, maxBytes);
277
+ if (read.status === "dataUrl") {
278
+ const base64 = read.dataUrl.slice(read.dataUrl.indexOf(",") + 1);
279
+ return { status: "ok", buffer: Buffer.from(base64, "base64") };
280
+ }
281
+ return read.status === "buffer"
282
+ ? { status: "ok", buffer: read.buffer }
283
+ : read;
111
284
  }
112
285
  /**
113
- * Turn the caller's images and video into `data:` URLs, or say why not.
114
- * Runs before any request, so every refusal costs nothing and none is retried.
286
+ * Turn the caller's images and video into `data:` URLs, and an audio clip into
287
+ * `{ data, format }`, or say why not. Runs before any request, so every
288
+ * refusal costs nothing and none is retried.
115
289
  */
116
290
  export async function prepareDecisionMedia(request, limits, vendor, alternatives) {
117
291
  const images = request.images ?? [];
118
- if (images.length === 0 && request.video === undefined) {
292
+ if (images.length === 0 &&
293
+ request.video === undefined &&
294
+ request.audio === undefined) {
119
295
  return { status: "prepared", media: undefined };
120
296
  }
297
+ const use = alternatives.length > 0
298
+ ? ` Use a decision provider that does: ${alternatives.join(", ")}.`
299
+ : "";
121
300
  if (!limits) {
122
- const use = alternatives.length > 0
123
- ? ` Use a decision provider that does: ${alternatives.join(", ")}.`
124
- : "";
301
+ const what = images.length === 0 && request.video === undefined
302
+ ? "audio"
303
+ : "images or video";
125
304
  return {
126
305
  status: "refused",
127
- message: `${vendor} does not accept images or video.${use}`,
306
+ message: `${vendor} does not accept ${what}.${use}`,
307
+ };
308
+ }
309
+ if (request.audio !== undefined && limits.audio === undefined) {
310
+ return {
311
+ status: "refused",
312
+ message: `${vendor} does not accept audio.${use}`,
128
313
  };
129
314
  }
130
315
  if (images.length > limits.maxImages) {
@@ -152,9 +337,26 @@ export async function prepareDecisionMedia(request, limits, vendor, alternatives
152
337
  }
153
338
  video = converted.dataUrl;
154
339
  }
155
- const bytes = prepared.reduce((sum, url) => sum + url.length, video?.length ?? 0);
340
+ let audio;
341
+ if (request.audio !== undefined && limits.audio !== undefined) {
342
+ const read = await readAudioSource(request.audio, "The audio", limits.maxRequestBytes);
343
+ if (read.status === "error") {
344
+ return { status: "refused", message: read.message };
345
+ }
346
+ const encoded = encodeAudio(read.buffer, limits.audio, "The audio");
347
+ if (encoded.status === "error") {
348
+ return { status: "refused", message: encoded.message };
349
+ }
350
+ audio = encoded.audio;
351
+ }
352
+ const bytes = prepared.reduce((sum, url) => sum + url.length, (video?.length ?? 0) + (audio?.data.length ?? 0));
156
353
  return {
157
354
  status: "prepared",
158
- media: { images: prepared, ...(video ? { video } : {}), bytes },
355
+ media: {
356
+ images: prepared,
357
+ ...(video ? { video } : {}),
358
+ ...(audio ? { audio } : {}),
359
+ bytes,
360
+ },
159
361
  };
160
362
  }
@@ -352,8 +352,10 @@ export function createTimeoutController(timeout, provider, operation) {
352
352
  // from here — the setTimeout callback runs in the async context captured
353
353
  // at schedule time, which is BEFORE the AI SDK span exists. Instead we
354
354
  // rely on the AI SDK propagating the TimeoutError through its recordSpan
355
- // wrapper, which sets span.status = ERROR + message. ContextEnricher's
356
- // SpanStatusCode.ERROR branch then surfaces level=ERROR + status_message.
355
+ // wrapper, which sets span.status = ERROR + message. NeuroLink's own
356
+ // spans the error passes through (withGeneration, withSpan) record it
357
+ // in markFailure as level=ERROR + status_message, and
358
+ // ExporterSpanProcessor exports them.
357
359
  controller.abort(new TimeoutError(`${provider} ${operation} operation timed out after ${timeout}`, timeoutMs, provider, operation));
358
360
  };
359
361
  let timer = setTimeout(fire, timeoutMs);
@@ -14,6 +14,7 @@
14
14
  *
15
15
  * See docs/features/livekit-voice-agent.md.
16
16
  */
17
+ import { randomUUID } from "node:crypto";
17
18
  import { z } from "zod";
18
19
  import { logger } from "../../utils/logger.js";
19
20
  import { findSchemaIssue, sanitizeToolParameters } from "./schemaSanitizer.js";
@@ -159,7 +160,10 @@ export async function buildRealtimeMcpTools(params) {
159
160
  args: args ?? {},
160
161
  });
161
162
  publishEvent("tool-start", { name: mcpTool.name });
162
- observe(() => onToolStart?.({ name: mcpTool.name, args: args ?? {} }));
163
+ // Concurrent calls of one tool can end in any order; the id lets the
164
+ // turn record give each its own result.
165
+ const callId = randomUUID();
166
+ observe(() => onToolStart?.({ name: mcpTool.name, args: args ?? {}, callId }));
163
167
  const startedAt = Date.now();
164
168
  try {
165
169
  // Third argument is RequestOptions; the SDK aborts the in-flight
@@ -182,6 +186,7 @@ export async function buildRealtimeMcpTools(params) {
182
186
  publishEvent("tool-result", { name: mcpTool.name, result });
183
187
  observe(() => onToolEnd?.({
184
188
  name: mcpTool.name,
189
+ callId,
185
190
  ...(result.isError === true
186
191
  ? { error: text }
187
192
  : { result: text }),
@@ -195,7 +200,11 @@ export async function buildRealtimeMcpTools(params) {
195
200
  error: error instanceof Error ? error.message : String(error),
196
201
  });
197
202
  publishEvent("tool-result", { name: mcpTool.name, result: null });
198
- observe(() => onToolEnd?.({ name: mcpTool.name, error: String(error) }));
203
+ observe(() => onToolEnd?.({
204
+ name: mcpTool.name,
205
+ callId,
206
+ error: String(error),
207
+ }));
199
208
  return `Tool ${mcpTool.name} failed: ${String(error)}`;
200
209
  }
201
210
  },
@@ -12,12 +12,30 @@
12
12
  * The shutdown flush is registered here, before the job lifecycle installs its
13
13
  * process reaper. LiveKit starts every shutdown callback in registration order
14
14
  * and the reaper exits the process about a second later, so the flush gets a
15
- * budget below that second and starts first.
15
+ * budget below that second and starts first. The same budget bounds delivery of
16
+ * the call's last traces, which go out while the last writes finish.
17
+ *
18
+ * Each turn is traced as it closes, never behind the store's one-at-a-time
19
+ * writes, so a slow or stuck store cannot delay or lose a trace.
20
+ *
21
+ * Tracing does not depend on memory. A call with no memory still gets a
22
+ * recorder that only traces its turns (`setUpRealtimeCallTracing`).
16
23
  *
17
24
  * See docs/features/livekit-voice-agent.md.
18
25
  */
19
26
  import type { JobContext } from "@livekit/agents";
20
- import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig } from "../../types/index.js";
27
+ import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig, VoiceTurnRecorder } from "../../types/index.js";
28
+ /**
29
+ * Prepare a recorder that only traces: for a call with no usable memory. Returns
30
+ * `undefined` when tracing is off. Without a verified binding there is no user,
31
+ * so the room is the session.
32
+ */
33
+ export declare function setUpRealtimeCallTracing(ctx: Pick<JobContext, "room" | "addShutdownCallback">, options?: {
34
+ model?: string;
35
+ backend?: RealtimeVoiceBackend;
36
+ settleMs?: number;
37
+ flushBudgetMs?: number;
38
+ }): VoiceTurnRecorder | undefined;
21
39
  /**
22
40
  * Verify the call's binding and prepare its recorder; `undefined` means "no
23
41
  * memory for this call". `call` names the model and the realtime backend the
@@ -12,13 +12,21 @@
12
12
  * The shutdown flush is registered here, before the job lifecycle installs its
13
13
  * process reaper. LiveKit starts every shutdown callback in registration order
14
14
  * and the reaper exits the process about a second later, so the flush gets a
15
- * budget below that second and starts first.
15
+ * budget below that second and starts first. The same budget bounds delivery of
16
+ * the call's last traces, which go out while the last writes finish.
17
+ *
18
+ * Each turn is traced as it closes, never behind the store's one-at-a-time
19
+ * writes, so a slow or stuck store cannot delay or lose a trace.
20
+ *
21
+ * Tracing does not depend on memory. A call with no memory still gets a
22
+ * recorder that only traces its turns (`setUpRealtimeCallTracing`).
16
23
  *
17
24
  * See docs/features/livekit-voice-agent.md.
18
25
  */
19
26
  import { context, trace } from "@opentelemetry/api";
20
27
  import { AIProviderName } from "../../constants/enums.js";
21
- import { setLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
28
+ import { flushOpenTelemetry, isOpenTelemetryInitialized, setLangfuseContext, } from "../../services/server/ai/observability/instrumentation.js";
29
+ import { isLangfuseEnabled } from "../../services/server/ai/observability/langfuseSetup.js";
22
30
  import { observe, observeTrace } from "../../telemetry/observation.js";
23
31
  import { tracers } from "../../telemetry/tracers.js";
24
32
  import { logger } from "../../utils/logger.js";
@@ -31,7 +39,7 @@ const TRACE_PROVIDER = {
31
39
  xai: AIProviderName.XAI,
32
40
  };
33
41
  function traceTurn(turn, call) {
34
- const { sessionId, userId, roomName, model, provider } = call;
42
+ const { sessionId, userId, roomName, model, provider = AIProviderName.VERTEX, } = call;
35
43
  try {
36
44
  setLangfuseContext({
37
45
  sessionId,
@@ -45,39 +53,62 @@ function traceTurn(turn, call) {
45
53
  const root = tracers.sdk.startSpan("realtime-turn", {
46
54
  startTime: turn.startTime,
47
55
  });
48
- observe(root, "agent", {
49
- input: turn.userMessage,
50
- output: turn.aiResponse,
51
- level,
52
- });
53
- observeTrace(root, {
54
- input: turn.userMessage,
55
- output: turn.aiResponse,
56
- });
57
- const inRoot = trace.setSpan(context.active(), root);
58
- const generation = tracers.generation.startSpan("realtime", { startTime: turn.startTime }, inRoot);
59
- observe(generation, "generation", {
60
- ...(model && { model }),
61
- input: [{ role: "user", content: turn.userMessage }],
62
- output: [{ role: "assistant", content: turn.aiResponse }],
63
- level,
64
- metadata: { provider },
65
- });
66
- generation.end(endTime);
67
- const events = turn.events ?? [];
68
- for (const start of events.filter((e) => e.type === "tool:start")) {
69
- const end = events.find((e) => e.type === "tool:end" &&
70
- e.tool === start.tool &&
71
- e.seq > start.seq);
72
- const tool = tracers.sdk.startSpan(String(start.tool), { startTime: start.timestamp }, inRoot);
73
- observe(tool, "tool", {
74
- input: start.args,
75
- output: end?.result ?? end?.error,
76
- level: end?.success === false ? "ERROR" : undefined,
56
+ // The root and each child end on every path, so a failure while
57
+ // describing one tool cannot leave the turn open and unexported.
58
+ try {
59
+ observe(root, "agent", {
60
+ input: turn.userMessage,
61
+ output: turn.aiResponse,
62
+ level,
77
63
  });
78
- tool.end(Math.min(end?.timestamp ?? Infinity, endTime.getTime()));
64
+ observeTrace(root, {
65
+ input: turn.userMessage,
66
+ output: turn.aiResponse,
67
+ });
68
+ const inRoot = trace.setSpan(context.active(), root);
69
+ const generation = tracers.generation.startSpan("realtime", { startTime: turn.startTime }, inRoot);
70
+ try {
71
+ observe(generation, "generation", {
72
+ ...(model && { model }),
73
+ input: [{ role: "user", content: turn.userMessage }],
74
+ output: [{ role: "assistant", content: turn.aiResponse }],
75
+ level,
76
+ metadata: { provider },
77
+ });
78
+ }
79
+ finally {
80
+ generation.end(endTime);
81
+ }
82
+ const events = turn.events ?? [];
83
+ const consumed = new Set();
84
+ for (const start of events.filter((e) => e.type === "tool:start")) {
85
+ // A call id pairs overlapping calls exactly; without one, each end
86
+ // is spent on the earliest unpaired start of the same tool.
87
+ const end = events.find((e) => e.type === "tool:end" &&
88
+ e.seq > start.seq &&
89
+ !consumed.has(e.seq) &&
90
+ (start.callId !== undefined
91
+ ? e.callId === start.callId
92
+ : e.tool === start.tool));
93
+ if (end) {
94
+ consumed.add(end.seq);
95
+ }
96
+ const tool = tracers.sdk.startSpan(String(start.tool), { startTime: start.timestamp }, inRoot);
97
+ try {
98
+ observe(tool, "tool", {
99
+ input: start.args,
100
+ output: end?.result ?? end?.error,
101
+ level: end?.success === false ? "ERROR" : undefined,
102
+ });
103
+ }
104
+ finally {
105
+ tool.end(Math.min(end?.timestamp ?? Infinity, endTime.getTime()));
106
+ }
107
+ }
108
+ }
109
+ finally {
110
+ root.end(endTime);
79
111
  }
80
- root.end(endTime);
81
112
  });
82
113
  }
83
114
  catch (error) {
@@ -86,6 +117,71 @@ function traceTurn(turn, call) {
86
117
  });
87
118
  }
88
119
  }
120
+ /** A tracer that spends one trace per turn id. */
121
+ function createTurnTracer(call) {
122
+ const tracedTurns = new Set();
123
+ return (turn) => {
124
+ if (!tracedTurns.has(turn.turnId)) {
125
+ tracedTurns.add(turn.turnId);
126
+ traceTurn(turn, call);
127
+ }
128
+ };
129
+ }
130
+ /** Deliver buffered traces, giving up after `budgetMs`: the worker is reaped about a second after shutdown starts. */
131
+ async function flushTraces(budgetMs) {
132
+ let timer;
133
+ try {
134
+ await Promise.race([
135
+ flushOpenTelemetry(),
136
+ new Promise((resolve) => {
137
+ timer = setTimeout(resolve, Math.max(0, budgetMs));
138
+ }),
139
+ ]);
140
+ }
141
+ catch (error) {
142
+ logger.debug("realtime.memory.traceFlushFailed", {
143
+ error: error instanceof Error ? error.message : String(error),
144
+ });
145
+ }
146
+ finally {
147
+ clearTimeout(timer);
148
+ }
149
+ }
150
+ /** Shutdown: close the open exchange, then deliver every trace while its writes finish, all within one budget. */
151
+ async function flushCall(recorder, budgetMs) {
152
+ // flush() closes, and so traces, the open exchange before it first waits.
153
+ const [stats] = await Promise.all([
154
+ recorder.flush(budgetMs),
155
+ flushTraces(budgetMs),
156
+ ]);
157
+ return stats;
158
+ }
159
+ /**
160
+ * Prepare a recorder that only traces: for a call with no usable memory. Returns
161
+ * `undefined` when tracing is off. Without a verified binding there is no user,
162
+ * so the room is the session.
163
+ */
164
+ export function setUpRealtimeCallTracing(ctx, options = {}) {
165
+ if (!isLangfuseEnabled() && !isOpenTelemetryInitialized()) {
166
+ return undefined;
167
+ }
168
+ const trace = createTurnTracer({
169
+ sessionId: ctx.room.name,
170
+ roomName: ctx.room.name,
171
+ model: options.model,
172
+ provider: TRACE_PROVIDER[options.backend ?? "gemini"],
173
+ });
174
+ const recorder = createVoiceTurnRecorder({
175
+ ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
176
+ onClosed: trace,
177
+ write: () => Promise.resolve(),
178
+ });
179
+ ctx.addShutdownCallback(async () => {
180
+ const stats = await flushCall(recorder, options.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
181
+ logger.info("realtime.trace.flushed", { ...stats });
182
+ });
183
+ return recorder;
184
+ }
89
185
  /**
90
186
  * Verify the call's binding and prepare its recorder; `undefined` means "no
91
187
  * memory for this call". `call` names the model and the realtime backend the
@@ -122,20 +218,17 @@ export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
122
218
  return undefined;
123
219
  }
124
220
  let publish;
125
- const tracedTurns = new Set();
221
+ const trace = createTurnTracer({
222
+ sessionId,
223
+ userId,
224
+ roomName: ctx.room.name,
225
+ model,
226
+ provider: TRACE_PROVIDER[backend],
227
+ });
126
228
  const recorder = createVoiceTurnRecorder({
127
229
  ...(config.settleMs !== undefined ? { settleMs: config.settleMs } : {}),
230
+ onClosed: trace,
128
231
  write: async (turn) => {
129
- if (!tracedTurns.has(turn.turnId)) {
130
- tracedTurns.add(turn.turnId);
131
- traceTurn(turn, {
132
- sessionId,
133
- userId,
134
- roomName: ctx.room.name,
135
- model,
136
- provider: TRACE_PROVIDER[backend],
137
- });
138
- }
139
232
  const result = await neurolink.recordConversationTurn({
140
233
  sessionId,
141
234
  userId,
@@ -160,7 +253,7 @@ export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
160
253
  },
161
254
  });
162
255
  ctx.addShutdownCallback(async () => {
163
- const stats = await recorder.flush(config.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
256
+ const stats = await flushCall(recorder, config.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
164
257
  logger.info("realtime.memory.flushed", { ...stats });
165
258
  });
166
259
  let seed = [];