@juspay/neurolink 13.5.0 → 13.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/adapters/video/videoAnalyzer.d.ts +1 -0
- package/dist/adapters/video/videoAnalyzer.js +4 -0
- package/dist/autoresearch/index.d.ts +1 -0
- package/dist/autoresearch/index.js +4 -0
- package/dist/autoresearch/worker.js +4 -0
- package/dist/browser/neurolink.min.js +476 -476
- package/dist/cli/commands/observability.js +11 -4
- package/dist/cli/commands/telemetry.js +36 -6
- package/dist/cli/utils/formatters.d.ts +17 -0
- package/dist/cli/utils/formatters.js +39 -0
- package/dist/constants/langfuse.d.ts +17 -1
- package/dist/constants/langfuse.js +23 -1
- package/dist/constants/telemetry.d.ts +6 -0
- package/dist/constants/telemetry.js +6 -0
- package/dist/core/baseProvider.d.ts +3 -1
- package/dist/core/baseProvider.js +30 -8
- package/dist/core/geminiLoopAdapter.js +19 -0
- package/dist/core/loopEngine.js +10 -7
- package/dist/core/modules/StreamHandler.js +5 -4
- package/dist/core/modules/TelemetryHandler.d.ts +5 -1
- package/dist/core/modules/TelemetryHandler.js +16 -0
- package/dist/core/modules/ToolsManager.js +14 -3
- package/dist/core/nativeGenerateLoop.d.ts +2 -0
- package/dist/core/nativeGenerateLoop.js +35 -9
- package/dist/evaluation/index.d.ts +1 -0
- package/dist/evaluation/index.js +4 -0
- package/dist/evaluation/ragasEvaluator.js +1 -0
- package/dist/mcp/externalServerManager.js +6 -2
- package/dist/mcp/toolRegistry.js +3 -0
- package/dist/neurolink.d.ts +16 -8
- package/dist/neurolink.js +115 -61
- package/dist/observability/FEATURE-STATUS.md +0 -4
- package/dist/observability/exporterRegistry.d.ts +6 -1
- package/dist/observability/exporterRegistry.js +18 -0
- package/dist/observability/exporterSpanProcessor.d.ts +20 -6
- package/dist/observability/exporterSpanProcessor.js +166 -34
- package/dist/observability/exporters/baseExporter.d.ts +22 -3
- package/dist/observability/exporters/baseExporter.js +87 -5
- package/dist/observability/exporters/langfuseExporter.d.ts +29 -3
- package/dist/observability/exporters/langfuseExporter.js +251 -41
- package/dist/observability/langfuseMasking.d.ts +1 -1
- package/dist/observability/langfuseMasking.js +1 -1
- package/dist/observability/otlpEncoding.d.ts +18 -5
- package/dist/observability/otlpEncoding.js +69 -12
- package/dist/observability/utils/spanSerializer.d.ts +3 -2
- package/dist/observability/utils/spanSerializer.js +5 -3
- package/dist/processors/base/BaseFileProcessor.d.ts +1 -0
- package/dist/processors/base/BaseFileProcessor.js +4 -0
- package/dist/processors/index.d.ts +1 -0
- package/dist/processors/index.js +4 -0
- package/dist/processors/registry/ProcessorRegistry.d.ts +1 -0
- package/dist/processors/registry/ProcessorRegistry.js +5 -0
- package/dist/providers/amazonBedrock/client.js +7 -5
- package/dist/providers/amazonSagemaker.js +32 -12
- package/dist/providers/anthropic/client.js +1 -0
- package/dist/providers/anthropic/loopAdapter.js +7 -3
- package/dist/providers/googleAiStudio/client.js +2 -0
- package/dist/providers/googleVertex/client.d.ts +6 -5
- package/dist/providers/googleVertex/client.js +46 -19
- package/dist/providers/openaiChatCompletionsBase.js +29 -18
- package/dist/proxy/codexUsage.js +59 -2
- package/dist/proxy/proxyTracer.d.ts +25 -1
- package/dist/proxy/proxyTracer.js +146 -22
- package/dist/proxy/proxyTranslationEngine.js +4 -2
- package/dist/proxy/sseInterceptor.js +6 -4
- package/dist/rag/chunkers/BaseChunker.js +1 -0
- package/dist/rag/index.d.ts +1 -0
- package/dist/rag/index.js +4 -0
- package/dist/rag/ragIntegration.js +1 -0
- package/dist/rag/reranker/reranker.js +2 -0
- package/dist/rag/retrieval/vectorQueryTool.js +1 -0
- package/dist/server/routes/codexProxyRoutes.js +3 -1
- package/dist/server/utils/validation.d.ts +3 -3
- package/dist/services/server/ai/observability/instrumentation.d.ts +8 -7
- package/dist/services/server/ai/observability/instrumentation.js +217 -47
- package/dist/services/server/ai/observability/langfuseSetup.d.ts +19 -1
- package/dist/services/server/ai/observability/langfuseSetup.js +62 -4
- package/dist/services/server/ai/observability/otlpSetup.d.ts +12 -2
- package/dist/services/server/ai/observability/otlpSetup.js +55 -4
- package/dist/telemetry/aiMetrics.d.ts +56 -0
- package/dist/telemetry/aiMetrics.js +229 -0
- package/dist/telemetry/attributes.d.ts +24 -1
- package/dist/telemetry/attributes.js +94 -9
- package/dist/telemetry/chatMessages.d.ts +6 -0
- package/dist/telemetry/chatMessages.js +37 -0
- package/dist/telemetry/index.d.ts +1 -1
- package/dist/telemetry/index.js +1 -1
- package/dist/telemetry/observation.d.ts +5 -5
- package/dist/telemetry/observation.js +128 -39
- package/dist/telemetry/spanSanitizer.d.ts +2 -2
- package/dist/telemetry/spanSanitizer.js +76 -8
- package/dist/telemetry/telemetryStarter.d.ts +16 -0
- package/dist/telemetry/telemetryStarter.js +28 -0
- package/dist/telemetry/traceContext.d.ts +3 -3
- package/dist/telemetry/traceContext.js +3 -3
- package/dist/telemetry/tracers.d.ts +13 -1
- package/dist/telemetry/tracers.js +49 -10
- package/dist/telemetry/withSpan.d.ts +19 -3
- package/dist/telemetry/withSpan.js +149 -20
- package/dist/tools/toolDiscovery.js +2 -1
- package/dist/tools/toolPolicy.js +22 -2
- package/dist/types/config.d.ts +19 -0
- package/dist/types/exporter.d.ts +98 -3
- package/dist/types/generate.d.ts +7 -1
- package/dist/types/livekit.d.ts +16 -1
- package/dist/types/loopEngine.d.ts +16 -1
- package/dist/types/observability.d.ts +38 -5
- package/dist/types/proxy.d.ts +2 -0
- package/dist/types/span.d.ts +26 -0
- package/dist/types/toolResolution.d.ts +5 -1
- package/dist/utils/timeout.js +4 -2
- package/dist/voice/livekit/realtimeMcpTools.js +11 -2
- package/dist/voice/livekit/realtimeMemory.d.ts +20 -2
- package/dist/voice/livekit/realtimeMemory.js +139 -46
- package/dist/voice/livekit/realtimeVoiceAgent.js +20 -8
- package/dist/voice/livekit/turnRecorder.js +22 -9
- package/dist/workflow/core/ensembleExecutor.js +2 -0
- package/dist/workflow/core/judgeScorer.js +1 -0
- package/dist/workflow/core/responseConditioner.js +1 -0
- package/dist/workflow/core/workflowRunner.js +6 -1
- package/dist/workflow/index.d.ts +1 -0
- package/dist/workflow/index.js +4 -0
- package/docs-site/static/search-index.json +32 -27
- package/package.json +21 -2
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* These configs are passed from the parent application (e.g., Lighthouse)
|
|
4
4
|
* to enable telemetry and observability features in Neurolink SDK
|
|
5
5
|
*/
|
|
6
|
-
import type { AttributeValue } from "@opentelemetry/api";
|
|
6
|
+
import type { AttributeValue, Counter, Histogram, Meter } from "@opentelemetry/api";
|
|
7
7
|
import type { JsonValue, RawUsageObject, Unknown } from "./common.js";
|
|
8
8
|
import type { SpanData } from "./span.js";
|
|
9
9
|
/**
|
|
@@ -37,20 +37,26 @@ export type LangfuseSpanAttributes = {
|
|
|
37
37
|
* @description Langfuse exporter options. Every field falls back to its `LANGFUSE_*` environment variable when omitted.
|
|
38
38
|
*/
|
|
39
39
|
export type NeurolinkLangfuseConfig = {
|
|
40
|
+
/** `false` keeps Langfuse export off although keys are present; any other value, or leaving it out, leaves it on. */
|
|
41
|
+
enabled?: boolean;
|
|
40
42
|
/** Project public key; defaults to `LANGFUSE_PUBLIC_KEY`. */
|
|
41
43
|
publicKey?: string;
|
|
42
44
|
/** Project secret key; defaults to `LANGFUSE_SECRET_KEY`. */
|
|
43
45
|
secretKey?: string;
|
|
44
46
|
/** Langfuse host; defaults to `LANGFUSE_BASE_URL`, then Langfuse Cloud. */
|
|
45
47
|
baseUrl?: string;
|
|
46
|
-
/** Environment name; defaults to `LANGFUSE_TRACING_ENVIRONMENT`. */
|
|
48
|
+
/** Environment name on Langfuse traces; defaults to `LANGFUSE_TRACING_ENVIRONMENT`. The OTLP resource's `deployment.environment` also falls back to `LANGFUSE_ENVIRONMENT`, then `PUBLIC_APP_ENVIRONMENT`. */
|
|
47
49
|
environment?: string;
|
|
48
|
-
/** Release identifier; defaults to `LANGFUSE_RELEASE`. */
|
|
50
|
+
/** Release identifier on Langfuse traces; defaults to `LANGFUSE_RELEASE`. The OTLP resource's `service.version` also falls back to `PUBLIC_APP_VERSION`. */
|
|
49
51
|
release?: string;
|
|
50
|
-
/** Applied to input, output and metadata before they
|
|
52
|
+
/** Applied to input, output and metadata before they are sent to Langfuse. It does not reach other exporters, the OTLP export, `getSpans()` or error text. */
|
|
51
53
|
mask?: (params: {
|
|
52
54
|
data: JsonValue;
|
|
53
55
|
}) => JsonValue;
|
|
56
|
+
/** Leaves input and output out of what is sent to Langfuse. It does not reach other exporters, the OTLP export, `getSpans()` or error text. */
|
|
57
|
+
redactIO?: boolean;
|
|
58
|
+
/** Milliseconds a flush or shutdown waits for the Langfuse exporter before it drops what is still in flight; defaults to 3000. */
|
|
59
|
+
flushBudgetMs?: number;
|
|
54
60
|
/** `"immediate"` sends every span as it ends; `"batched"` (default) queues them. */
|
|
55
61
|
exportMode?: "batched" | "immediate";
|
|
56
62
|
/** Spans queued before a batch is sent; defaults to 15. */
|
|
@@ -59,7 +65,26 @@ export type NeurolinkLangfuseConfig = {
|
|
|
59
65
|
flushInterval?: number;
|
|
60
66
|
};
|
|
61
67
|
/**
|
|
62
|
-
* @description
|
|
68
|
+
* @description Kind of connection counted by the `connections_total` metric.
|
|
69
|
+
*/
|
|
70
|
+
export type MetricConnectionType = "websocket" | "sse" | "http";
|
|
71
|
+
/**
|
|
72
|
+
* @description The OpenTelemetry instruments behind NeuroLink's `ai_*`, `mcp_*`, `connections_total`, `response_time_ms` and `custom_*` metrics, created on one meter.
|
|
73
|
+
*/
|
|
74
|
+
export type AiMetricInstruments = {
|
|
75
|
+
meter: Meter;
|
|
76
|
+
requests: Counter;
|
|
77
|
+
duration: Histogram;
|
|
78
|
+
tokens: Counter;
|
|
79
|
+
cost: Counter;
|
|
80
|
+
providerErrors: Counter;
|
|
81
|
+
mcpToolCalls: Counter;
|
|
82
|
+
connections: Counter;
|
|
83
|
+
responseTime: Histogram;
|
|
84
|
+
custom: Map<string, Counter | Histogram>;
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* @description Observability configuration for the NeuroLink SDK. Langfuse turns on automatically when `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` are set and `LANGFUSE_ENABLED` is not `false`, `0`, `off` or `no`; `false` turns it off.
|
|
63
88
|
*/
|
|
64
89
|
export type ObservabilityConfig = {
|
|
65
90
|
/** `false` disables Langfuse, `true` or an options object enables it. */
|
|
@@ -387,12 +412,20 @@ export type HealthMetrics = {
|
|
|
387
412
|
export type SpanOptions = {
|
|
388
413
|
name: string;
|
|
389
414
|
tracer: import("@opentelemetry/api").Tracer;
|
|
415
|
+
/** Marks the span as an observation of this type when it starts, which is what sends it to the exporters. Without it a span is exported only if its own code marks it with `observe`. */
|
|
416
|
+
observationType?: ObservationType;
|
|
390
417
|
kind?: import("@opentelemetry/api").SpanKind;
|
|
391
418
|
attributes?: Record<string, string | number | boolean | undefined>;
|
|
392
419
|
/** Marks a thrown error as an expected skip whose Langfuse level the span set itself, so `withSpan` keeps it instead of raising ERROR. */
|
|
393
420
|
expectedFailure?: (error: unknown) => boolean;
|
|
394
421
|
/** Keeps the span out of the active context, so spans started inside `fn` parent to the nearest ancestor that is exported. */
|
|
395
422
|
detached?: boolean;
|
|
423
|
+
/**
|
|
424
|
+
* With `detached`, lets the span be the active, exported root of its trace
|
|
425
|
+
* when no span is active, so what it wraps stays in one trace instead of
|
|
426
|
+
* each child starting its own. With an active ancestor it stays detached.
|
|
427
|
+
*/
|
|
428
|
+
rootWhenOrphaned?: boolean;
|
|
396
429
|
};
|
|
397
430
|
/** Trace + parent span IDs used to correlate metric records with spans. */
|
|
398
431
|
export type MetricsTraceContext = {
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -2478,6 +2478,8 @@ export type CodexStreamEvidence = {
|
|
|
2478
2478
|
terminalBytes: number;
|
|
2479
2479
|
/** Malformed, oversized or undispatched frames prevent proving output absence. */
|
|
2480
2480
|
observationIncomplete?: boolean;
|
|
2481
|
+
/** Output items past the recorded-size cap were left out of `response.output`. */
|
|
2482
|
+
outputItemsOmitted?: boolean;
|
|
2481
2483
|
firstUsefulOutputAt?: number;
|
|
2482
2484
|
firstUsefulOutputEvent?: string;
|
|
2483
2485
|
errorType?: string;
|
package/dist/types/span.d.ts
CHANGED
|
@@ -141,8 +141,24 @@ export type SpanAttributes = {
|
|
|
141
141
|
export type SpanEvent = {
|
|
142
142
|
name: string;
|
|
143
143
|
timestamp: string;
|
|
144
|
+
/** When the event happened as nanoseconds since the Unix epoch (decimal string); finer than `timestamp`. */
|
|
145
|
+
timestampUnixNano?: string;
|
|
144
146
|
attributes?: Record<string, unknown>;
|
|
145
147
|
};
|
|
148
|
+
/**
|
|
149
|
+
* Instrumentation scope that created a span
|
|
150
|
+
*/
|
|
151
|
+
export type SpanScope = {
|
|
152
|
+
name: string;
|
|
153
|
+
version?: string;
|
|
154
|
+
};
|
|
155
|
+
/**
|
|
156
|
+
* The fields an OpenTelemetry SDK 1.x span exposes where 2.x uses `parentSpanContext` and `instrumentationScope`
|
|
157
|
+
*/
|
|
158
|
+
export type LegacyReadableSpan = {
|
|
159
|
+
parentSpanId?: string;
|
|
160
|
+
instrumentationLibrary?: SpanScope;
|
|
161
|
+
};
|
|
146
162
|
/**
|
|
147
163
|
* Link to related span
|
|
148
164
|
*/
|
|
@@ -169,8 +185,18 @@ export type SpanData = {
|
|
|
169
185
|
startTime: string;
|
|
170
186
|
/** End timestamp (ISO 8601) */
|
|
171
187
|
endTime?: string;
|
|
188
|
+
/** Start as nanoseconds since the Unix epoch (decimal string); finer than `startTime` */
|
|
189
|
+
startTimeUnixNano?: string;
|
|
190
|
+
/** End as nanoseconds since the Unix epoch (decimal string); finer than `endTime` */
|
|
191
|
+
endTimeUnixNano?: string;
|
|
172
192
|
/** Duration in milliseconds */
|
|
173
193
|
durationMs?: number;
|
|
194
|
+
/** OpenTelemetry span kind: 0 internal, 1 server, 2 client, 3 producer, 4 consumer */
|
|
195
|
+
kind?: number;
|
|
196
|
+
/** Attributes of the resource that produced the span, such as `service.name` */
|
|
197
|
+
resource?: SpanAttributes;
|
|
198
|
+
/** Instrumentation scope that created the span */
|
|
199
|
+
scope?: SpanScope;
|
|
174
200
|
/** Span status */
|
|
175
201
|
status: SpanStatus;
|
|
176
202
|
/** Status message (for errors) */
|
|
@@ -35,8 +35,12 @@ export type ResolvedToolPolicy = {
|
|
|
35
35
|
includeBound?: string[];
|
|
36
36
|
/** Denylist of tool-name patterns (exact names or `*` globs), applied after include. */
|
|
37
37
|
exclude: string[];
|
|
38
|
-
/** Defer
|
|
38
|
+
/** Defer tool schemas behind the search_tools meta-tool. */
|
|
39
39
|
discovery: boolean;
|
|
40
|
+
/** Which tools discovery may defer. Absent means "external". */
|
|
41
|
+
discoveryScope?: "external" | "all";
|
|
42
|
+
/** Tool-name patterns (exact names or `*` globs) discovery never defers. Absent means none. */
|
|
43
|
+
discoveryAlwaysVisible?: string[];
|
|
40
44
|
/** Which option/config sources contributed to this policy (telemetry/debugging). */
|
|
41
45
|
sources: string[];
|
|
42
46
|
};
|
package/dist/utils/timeout.js
CHANGED
|
@@ -352,8 +352,10 @@ export function createTimeoutController(timeout, provider, operation) {
|
|
|
352
352
|
// from here — the setTimeout callback runs in the async context captured
|
|
353
353
|
// at schedule time, which is BEFORE the AI SDK span exists. Instead we
|
|
354
354
|
// rely on the AI SDK propagating the TimeoutError through its recordSpan
|
|
355
|
-
// wrapper, which sets span.status = ERROR + message.
|
|
356
|
-
//
|
|
355
|
+
// wrapper, which sets span.status = ERROR + message. NeuroLink's own
|
|
356
|
+
// spans the error passes through (withGeneration, withSpan) record it
|
|
357
|
+
// in markFailure as level=ERROR + status_message, and
|
|
358
|
+
// ExporterSpanProcessor exports them.
|
|
357
359
|
controller.abort(new TimeoutError(`${provider} ${operation} operation timed out after ${timeout}`, timeoutMs, provider, operation));
|
|
358
360
|
};
|
|
359
361
|
let timer = setTimeout(fire, timeoutMs);
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* See docs/features/livekit-voice-agent.md.
|
|
16
16
|
*/
|
|
17
|
+
import { randomUUID } from "node:crypto";
|
|
17
18
|
import { z } from "zod";
|
|
18
19
|
import { logger } from "../../utils/logger.js";
|
|
19
20
|
import { findSchemaIssue, sanitizeToolParameters } from "./schemaSanitizer.js";
|
|
@@ -159,7 +160,10 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
159
160
|
args: args ?? {},
|
|
160
161
|
});
|
|
161
162
|
publishEvent("tool-start", { name: mcpTool.name });
|
|
162
|
-
|
|
163
|
+
// Concurrent calls of one tool can end in any order; the id lets the
|
|
164
|
+
// turn record give each its own result.
|
|
165
|
+
const callId = randomUUID();
|
|
166
|
+
observe(() => onToolStart?.({ name: mcpTool.name, args: args ?? {}, callId }));
|
|
163
167
|
const startedAt = Date.now();
|
|
164
168
|
try {
|
|
165
169
|
// Third argument is RequestOptions; the SDK aborts the in-flight
|
|
@@ -182,6 +186,7 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
182
186
|
publishEvent("tool-result", { name: mcpTool.name, result });
|
|
183
187
|
observe(() => onToolEnd?.({
|
|
184
188
|
name: mcpTool.name,
|
|
189
|
+
callId,
|
|
185
190
|
...(result.isError === true
|
|
186
191
|
? { error: text }
|
|
187
192
|
: { result: text }),
|
|
@@ -195,7 +200,11 @@ export async function buildRealtimeMcpTools(params) {
|
|
|
195
200
|
error: error instanceof Error ? error.message : String(error),
|
|
196
201
|
});
|
|
197
202
|
publishEvent("tool-result", { name: mcpTool.name, result: null });
|
|
198
|
-
observe(() => onToolEnd?.({
|
|
203
|
+
observe(() => onToolEnd?.({
|
|
204
|
+
name: mcpTool.name,
|
|
205
|
+
callId,
|
|
206
|
+
error: String(error),
|
|
207
|
+
}));
|
|
199
208
|
return `Tool ${mcpTool.name} failed: ${String(error)}`;
|
|
200
209
|
}
|
|
201
210
|
},
|
|
@@ -12,12 +12,30 @@
|
|
|
12
12
|
* The shutdown flush is registered here, before the job lifecycle installs its
|
|
13
13
|
* process reaper. LiveKit starts every shutdown callback in registration order
|
|
14
14
|
* and the reaper exits the process about a second later, so the flush gets a
|
|
15
|
-
* budget below that second and starts first.
|
|
15
|
+
* budget below that second and starts first. The same budget bounds delivery of
|
|
16
|
+
* the call's last traces, which go out while the last writes finish.
|
|
17
|
+
*
|
|
18
|
+
* Each turn is traced as it closes, never behind the store's one-at-a-time
|
|
19
|
+
* writes, so a slow or stuck store cannot delay or lose a trace.
|
|
20
|
+
*
|
|
21
|
+
* Tracing does not depend on memory. A call with no memory still gets a
|
|
22
|
+
* recorder that only traces its turns (`setUpRealtimeCallTracing`).
|
|
16
23
|
*
|
|
17
24
|
* See docs/features/livekit-voice-agent.md.
|
|
18
25
|
*/
|
|
19
26
|
import type { JobContext } from "@livekit/agents";
|
|
20
|
-
import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig } from "../../types/index.js";
|
|
27
|
+
import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig, VoiceTurnRecorder } from "../../types/index.js";
|
|
28
|
+
/**
|
|
29
|
+
* Prepare a recorder that only traces: for a call with no usable memory. Returns
|
|
30
|
+
* `undefined` when tracing is off. Without a verified binding there is no user,
|
|
31
|
+
* so the room is the session.
|
|
32
|
+
*/
|
|
33
|
+
export declare function setUpRealtimeCallTracing(ctx: Pick<JobContext, "room" | "addShutdownCallback">, options?: {
|
|
34
|
+
model?: string;
|
|
35
|
+
backend?: RealtimeVoiceBackend;
|
|
36
|
+
settleMs?: number;
|
|
37
|
+
flushBudgetMs?: number;
|
|
38
|
+
}): VoiceTurnRecorder | undefined;
|
|
21
39
|
/**
|
|
22
40
|
* Verify the call's binding and prepare its recorder; `undefined` means "no
|
|
23
41
|
* memory for this call". `call` names the model and the realtime backend the
|
|
@@ -12,13 +12,21 @@
|
|
|
12
12
|
* The shutdown flush is registered here, before the job lifecycle installs its
|
|
13
13
|
* process reaper. LiveKit starts every shutdown callback in registration order
|
|
14
14
|
* and the reaper exits the process about a second later, so the flush gets a
|
|
15
|
-
* budget below that second and starts first.
|
|
15
|
+
* budget below that second and starts first. The same budget bounds delivery of
|
|
16
|
+
* the call's last traces, which go out while the last writes finish.
|
|
17
|
+
*
|
|
18
|
+
* Each turn is traced as it closes, never behind the store's one-at-a-time
|
|
19
|
+
* writes, so a slow or stuck store cannot delay or lose a trace.
|
|
20
|
+
*
|
|
21
|
+
* Tracing does not depend on memory. A call with no memory still gets a
|
|
22
|
+
* recorder that only traces its turns (`setUpRealtimeCallTracing`).
|
|
16
23
|
*
|
|
17
24
|
* See docs/features/livekit-voice-agent.md.
|
|
18
25
|
*/
|
|
19
26
|
import { context, trace } from "@opentelemetry/api";
|
|
20
27
|
import { AIProviderName } from "../../constants/enums.js";
|
|
21
|
-
import { setLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
|
|
28
|
+
import { flushOpenTelemetry, isOpenTelemetryInitialized, setLangfuseContext, } from "../../services/server/ai/observability/instrumentation.js";
|
|
29
|
+
import { isLangfuseEnabled } from "../../services/server/ai/observability/langfuseSetup.js";
|
|
22
30
|
import { observe, observeTrace } from "../../telemetry/observation.js";
|
|
23
31
|
import { tracers } from "../../telemetry/tracers.js";
|
|
24
32
|
import { logger } from "../../utils/logger.js";
|
|
@@ -31,7 +39,7 @@ const TRACE_PROVIDER = {
|
|
|
31
39
|
xai: AIProviderName.XAI,
|
|
32
40
|
};
|
|
33
41
|
function traceTurn(turn, call) {
|
|
34
|
-
const { sessionId, userId, roomName, model, provider } = call;
|
|
42
|
+
const { sessionId, userId, roomName, model, provider = AIProviderName.VERTEX, } = call;
|
|
35
43
|
try {
|
|
36
44
|
setLangfuseContext({
|
|
37
45
|
sessionId,
|
|
@@ -45,39 +53,62 @@ function traceTurn(turn, call) {
|
|
|
45
53
|
const root = tracers.sdk.startSpan("realtime-turn", {
|
|
46
54
|
startTime: turn.startTime,
|
|
47
55
|
});
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
output: turn.aiResponse,
|
|
56
|
-
});
|
|
57
|
-
const inRoot = trace.setSpan(context.active(), root);
|
|
58
|
-
const generation = tracers.generation.startSpan("realtime", { startTime: turn.startTime }, inRoot);
|
|
59
|
-
observe(generation, "generation", {
|
|
60
|
-
...(model && { model }),
|
|
61
|
-
input: [{ role: "user", content: turn.userMessage }],
|
|
62
|
-
output: [{ role: "assistant", content: turn.aiResponse }],
|
|
63
|
-
level,
|
|
64
|
-
metadata: { provider },
|
|
65
|
-
});
|
|
66
|
-
generation.end(endTime);
|
|
67
|
-
const events = turn.events ?? [];
|
|
68
|
-
for (const start of events.filter((e) => e.type === "tool:start")) {
|
|
69
|
-
const end = events.find((e) => e.type === "tool:end" &&
|
|
70
|
-
e.tool === start.tool &&
|
|
71
|
-
e.seq > start.seq);
|
|
72
|
-
const tool = tracers.sdk.startSpan(String(start.tool), { startTime: start.timestamp }, inRoot);
|
|
73
|
-
observe(tool, "tool", {
|
|
74
|
-
input: start.args,
|
|
75
|
-
output: end?.result ?? end?.error,
|
|
76
|
-
level: end?.success === false ? "ERROR" : undefined,
|
|
56
|
+
// The root and each child end on every path, so a failure while
|
|
57
|
+
// describing one tool cannot leave the turn open and unexported.
|
|
58
|
+
try {
|
|
59
|
+
observe(root, "agent", {
|
|
60
|
+
input: turn.userMessage,
|
|
61
|
+
output: turn.aiResponse,
|
|
62
|
+
level,
|
|
77
63
|
});
|
|
78
|
-
|
|
64
|
+
observeTrace(root, {
|
|
65
|
+
input: turn.userMessage,
|
|
66
|
+
output: turn.aiResponse,
|
|
67
|
+
});
|
|
68
|
+
const inRoot = trace.setSpan(context.active(), root);
|
|
69
|
+
const generation = tracers.generation.startSpan("realtime", { startTime: turn.startTime }, inRoot);
|
|
70
|
+
try {
|
|
71
|
+
observe(generation, "generation", {
|
|
72
|
+
...(model && { model }),
|
|
73
|
+
input: [{ role: "user", content: turn.userMessage }],
|
|
74
|
+
output: [{ role: "assistant", content: turn.aiResponse }],
|
|
75
|
+
level,
|
|
76
|
+
metadata: { provider },
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
generation.end(endTime);
|
|
81
|
+
}
|
|
82
|
+
const events = turn.events ?? [];
|
|
83
|
+
const consumed = new Set();
|
|
84
|
+
for (const start of events.filter((e) => e.type === "tool:start")) {
|
|
85
|
+
// A call id pairs overlapping calls exactly; without one, each end
|
|
86
|
+
// is spent on the earliest unpaired start of the same tool.
|
|
87
|
+
const end = events.find((e) => e.type === "tool:end" &&
|
|
88
|
+
e.seq > start.seq &&
|
|
89
|
+
!consumed.has(e.seq) &&
|
|
90
|
+
(start.callId !== undefined
|
|
91
|
+
? e.callId === start.callId
|
|
92
|
+
: e.tool === start.tool));
|
|
93
|
+
if (end) {
|
|
94
|
+
consumed.add(end.seq);
|
|
95
|
+
}
|
|
96
|
+
const tool = tracers.sdk.startSpan(String(start.tool), { startTime: start.timestamp }, inRoot);
|
|
97
|
+
try {
|
|
98
|
+
observe(tool, "tool", {
|
|
99
|
+
input: start.args,
|
|
100
|
+
output: end?.result ?? end?.error,
|
|
101
|
+
level: end?.success === false ? "ERROR" : undefined,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
tool.end(Math.min(end?.timestamp ?? Infinity, endTime.getTime()));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
root.end(endTime);
|
|
79
111
|
}
|
|
80
|
-
root.end(endTime);
|
|
81
112
|
});
|
|
82
113
|
}
|
|
83
114
|
catch (error) {
|
|
@@ -86,6 +117,71 @@ function traceTurn(turn, call) {
|
|
|
86
117
|
});
|
|
87
118
|
}
|
|
88
119
|
}
|
|
120
|
+
/** A tracer that spends one trace per turn id. */
|
|
121
|
+
function createTurnTracer(call) {
|
|
122
|
+
const tracedTurns = new Set();
|
|
123
|
+
return (turn) => {
|
|
124
|
+
if (!tracedTurns.has(turn.turnId)) {
|
|
125
|
+
tracedTurns.add(turn.turnId);
|
|
126
|
+
traceTurn(turn, call);
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/** Deliver buffered traces, giving up after `budgetMs`: the worker is reaped about a second after shutdown starts. */
|
|
131
|
+
async function flushTraces(budgetMs) {
|
|
132
|
+
let timer;
|
|
133
|
+
try {
|
|
134
|
+
await Promise.race([
|
|
135
|
+
flushOpenTelemetry(),
|
|
136
|
+
new Promise((resolve) => {
|
|
137
|
+
timer = setTimeout(resolve, Math.max(0, budgetMs));
|
|
138
|
+
}),
|
|
139
|
+
]);
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
logger.debug("realtime.memory.traceFlushFailed", {
|
|
143
|
+
error: error instanceof Error ? error.message : String(error),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
clearTimeout(timer);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/** Shutdown: close the open exchange, then deliver every trace while its writes finish, all within one budget. */
|
|
151
|
+
async function flushCall(recorder, budgetMs) {
|
|
152
|
+
// flush() closes, and so traces, the open exchange before it first waits.
|
|
153
|
+
const [stats] = await Promise.all([
|
|
154
|
+
recorder.flush(budgetMs),
|
|
155
|
+
flushTraces(budgetMs),
|
|
156
|
+
]);
|
|
157
|
+
return stats;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Prepare a recorder that only traces: for a call with no usable memory. Returns
|
|
161
|
+
* `undefined` when tracing is off. Without a verified binding there is no user,
|
|
162
|
+
* so the room is the session.
|
|
163
|
+
*/
|
|
164
|
+
export function setUpRealtimeCallTracing(ctx, options = {}) {
|
|
165
|
+
if (!isLangfuseEnabled() && !isOpenTelemetryInitialized()) {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
const trace = createTurnTracer({
|
|
169
|
+
sessionId: ctx.room.name,
|
|
170
|
+
roomName: ctx.room.name,
|
|
171
|
+
model: options.model,
|
|
172
|
+
provider: TRACE_PROVIDER[options.backend ?? "gemini"],
|
|
173
|
+
});
|
|
174
|
+
const recorder = createVoiceTurnRecorder({
|
|
175
|
+
...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
|
|
176
|
+
onClosed: trace,
|
|
177
|
+
write: () => Promise.resolve(),
|
|
178
|
+
});
|
|
179
|
+
ctx.addShutdownCallback(async () => {
|
|
180
|
+
const stats = await flushCall(recorder, options.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
|
|
181
|
+
logger.info("realtime.trace.flushed", { ...stats });
|
|
182
|
+
});
|
|
183
|
+
return recorder;
|
|
184
|
+
}
|
|
89
185
|
/**
|
|
90
186
|
* Verify the call's binding and prepare its recorder; `undefined` means "no
|
|
91
187
|
* memory for this call". `call` names the model and the realtime backend the
|
|
@@ -122,20 +218,17 @@ export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
|
|
|
122
218
|
return undefined;
|
|
123
219
|
}
|
|
124
220
|
let publish;
|
|
125
|
-
const
|
|
221
|
+
const trace = createTurnTracer({
|
|
222
|
+
sessionId,
|
|
223
|
+
userId,
|
|
224
|
+
roomName: ctx.room.name,
|
|
225
|
+
model,
|
|
226
|
+
provider: TRACE_PROVIDER[backend],
|
|
227
|
+
});
|
|
126
228
|
const recorder = createVoiceTurnRecorder({
|
|
127
229
|
...(config.settleMs !== undefined ? { settleMs: config.settleMs } : {}),
|
|
230
|
+
onClosed: trace,
|
|
128
231
|
write: async (turn) => {
|
|
129
|
-
if (!tracedTurns.has(turn.turnId)) {
|
|
130
|
-
tracedTurns.add(turn.turnId);
|
|
131
|
-
traceTurn(turn, {
|
|
132
|
-
sessionId,
|
|
133
|
-
userId,
|
|
134
|
-
roomName: ctx.room.name,
|
|
135
|
-
model,
|
|
136
|
-
provider: TRACE_PROVIDER[backend],
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
232
|
const result = await neurolink.recordConversationTurn({
|
|
140
233
|
sessionId,
|
|
141
234
|
userId,
|
|
@@ -160,7 +253,7 @@ export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
|
|
|
160
253
|
},
|
|
161
254
|
});
|
|
162
255
|
ctx.addShutdownCallback(async () => {
|
|
163
|
-
const stats = await recorder
|
|
256
|
+
const stats = await flushCall(recorder, config.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
|
|
164
257
|
logger.info("realtime.memory.flushed", { ...stats });
|
|
165
258
|
});
|
|
166
259
|
let seed = [];
|
|
@@ -23,7 +23,7 @@ import { createRealtimeLlm, requireXaiApiKey } from "./realtimeModel.js";
|
|
|
23
23
|
import { readCallContextFromRoom } from "./roomContext.js";
|
|
24
24
|
import { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
|
|
25
25
|
import { buildRealtimeMcpTools } from "./realtimeMcpTools.js";
|
|
26
|
-
import { setUpRealtimeCallMemory } from "./realtimeMemory.js";
|
|
26
|
+
import { setUpRealtimeCallMemory, setUpRealtimeCallTracing, } from "./realtimeMemory.js";
|
|
27
27
|
const realtimeLogEventSchema = z.object({
|
|
28
28
|
level: z.enum(["debug", "info", "warn", "error"]),
|
|
29
29
|
message: z.string(),
|
|
@@ -219,6 +219,18 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
219
219
|
model: cfg.model,
|
|
220
220
|
backend: cfg.backend,
|
|
221
221
|
});
|
|
222
|
+
// Tracing does not depend on memory: a call without it is still traced.
|
|
223
|
+
const recorder = memory?.recorder ??
|
|
224
|
+
setUpRealtimeCallTracing(ctx, {
|
|
225
|
+
model: cfg.model,
|
|
226
|
+
backend: cfg.backend,
|
|
227
|
+
...(memoryConfig?.settleMs !== undefined
|
|
228
|
+
? { settleMs: memoryConfig.settleMs }
|
|
229
|
+
: {}),
|
|
230
|
+
...(memoryConfig?.flushBudgetMs !== undefined
|
|
231
|
+
? { flushBudgetMs: memoryConfig.flushBudgetMs }
|
|
232
|
+
: {}),
|
|
233
|
+
});
|
|
222
234
|
await installRealtimeJobLifecycle(ctx, cfg);
|
|
223
235
|
const { voice } = await import("@livekit/agents");
|
|
224
236
|
const realtimeLlm = await createRealtimeLlm(cfg);
|
|
@@ -249,10 +261,10 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
249
261
|
xContext,
|
|
250
262
|
publishEvent: bridge.publishEvent,
|
|
251
263
|
requestConfirmation: bridge.requestConfirmation,
|
|
252
|
-
...(
|
|
264
|
+
...(recorder
|
|
253
265
|
? {
|
|
254
|
-
onToolStart: (call) =>
|
|
255
|
-
onToolEnd: (call) =>
|
|
266
|
+
onToolStart: (call) => recorder.toolStart(call),
|
|
267
|
+
onToolEnd: (call) => recorder.toolEnd(call),
|
|
256
268
|
}
|
|
257
269
|
: {}),
|
|
258
270
|
});
|
|
@@ -319,7 +331,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
319
331
|
});
|
|
320
332
|
}
|
|
321
333
|
bridge.publishEvent("text", { delta: chunk });
|
|
322
|
-
|
|
334
|
+
recorder?.assistantDelta(chunk);
|
|
323
335
|
}
|
|
324
336
|
}
|
|
325
337
|
}
|
|
@@ -358,7 +370,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
358
370
|
text: ev.transcript,
|
|
359
371
|
final: ev.isFinal,
|
|
360
372
|
});
|
|
361
|
-
|
|
373
|
+
recorder?.userTranscript({
|
|
362
374
|
text: ev.transcript,
|
|
363
375
|
final: ev.isFinal,
|
|
364
376
|
});
|
|
@@ -379,7 +391,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
379
391
|
// The id the stored messages carry, so a client can merge its live
|
|
380
392
|
// copy of the turn with history instead of showing it twice.
|
|
381
393
|
bridge.publishEvent("done", memory ? { turnId: ev.item.id } : {});
|
|
382
|
-
|
|
394
|
+
recorder?.assistantMessage({
|
|
383
395
|
id: ev.item.id,
|
|
384
396
|
text: ev.item.textContent,
|
|
385
397
|
interrupted: ev.item.interrupted,
|
|
@@ -398,7 +410,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
|
|
|
398
410
|
bridge.publishEvent("status", { state: "listening" });
|
|
399
411
|
if (cfg.greeting.trim().length > 0) {
|
|
400
412
|
logger.info("realtime.session.greeting", { ms: sinceConnect() });
|
|
401
|
-
|
|
413
|
+
recorder?.expectGreeting();
|
|
402
414
|
session.generateReply({ instructions: cfg.greeting });
|
|
403
415
|
}
|
|
404
416
|
}
|
|
@@ -102,6 +102,18 @@ async function writeWithRetry(record, context) {
|
|
|
102
102
|
/* a failure sink must never break the call */
|
|
103
103
|
}
|
|
104
104
|
}
|
|
105
|
+
/** The stored form of a closed exchange. */
|
|
106
|
+
function turnRecord(exchange) {
|
|
107
|
+
return {
|
|
108
|
+
turnId: exchange.turnId,
|
|
109
|
+
userMessage: exchange.user,
|
|
110
|
+
aiResponse: exchange.assistant,
|
|
111
|
+
interrupted: exchange.interrupted,
|
|
112
|
+
callEnded: exchange.callEnded,
|
|
113
|
+
startTime: new Date(exchange.startedAt),
|
|
114
|
+
...(exchange.events.length > 0 ? { events: exchange.events } : {}),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
105
117
|
/** Create a recorder for one call. */
|
|
106
118
|
export function createVoiceTurnRecorder(options) {
|
|
107
119
|
const settleMs = options.settleMs ?? DEFAULT_SETTLE_MS;
|
|
@@ -165,15 +177,14 @@ export function createVoiceTurnRecorder(options) {
|
|
|
165
177
|
});
|
|
166
178
|
}
|
|
167
179
|
function finishExchange(exchange) {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
});
|
|
180
|
+
const record = turnRecord(exchange);
|
|
181
|
+
try {
|
|
182
|
+
options.onClosed?.(record);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
/* an observer of closed exchanges must never break the call */
|
|
186
|
+
}
|
|
187
|
+
enqueue(record);
|
|
177
188
|
}
|
|
178
189
|
function syntheticTurnId() {
|
|
179
190
|
syntheticTurns += 1;
|
|
@@ -297,6 +308,7 @@ export function createVoiceTurnRecorder(options) {
|
|
|
297
308
|
open.events.push(nextEvent("tool:start", {
|
|
298
309
|
tool: call.name,
|
|
299
310
|
toolName: call.name,
|
|
311
|
+
...(call.callId !== undefined ? { callId: call.callId } : {}),
|
|
300
312
|
...(call.args !== undefined
|
|
301
313
|
? { args: boundedPayload(call.args) }
|
|
302
314
|
: {}),
|
|
@@ -309,6 +321,7 @@ export function createVoiceTurnRecorder(options) {
|
|
|
309
321
|
open.events.push(nextEvent("tool:end", {
|
|
310
322
|
tool: call.name,
|
|
311
323
|
toolName: call.name,
|
|
324
|
+
...(call.callId !== undefined ? { callId: call.callId } : {}),
|
|
312
325
|
success: call.error === undefined,
|
|
313
326
|
...(call.error !== undefined
|
|
314
327
|
? { error: boundedText(call.error) }
|
|
@@ -23,6 +23,7 @@ export async function executeEnsemble(options) {
|
|
|
23
23
|
return withSpan({
|
|
24
24
|
name: "neurolink.workflow.ensemble.execute",
|
|
25
25
|
tracer: tracers.workflow,
|
|
26
|
+
observationType: "span",
|
|
26
27
|
attributes: {
|
|
27
28
|
"workflow.model_count": options.models.length,
|
|
28
29
|
"workflow.parallelism": options.executionConfig?.parallelism ?? 10,
|
|
@@ -318,6 +319,7 @@ export async function executeModelGroups(groups, prompt, _executionConfig, syste
|
|
|
318
319
|
return withSpan({
|
|
319
320
|
name: "neurolink.workflow.layers.execute",
|
|
320
321
|
tracer: tracers.workflow,
|
|
322
|
+
observationType: "span",
|
|
321
323
|
attributes: {
|
|
322
324
|
"workflow.group_count": groups.length,
|
|
323
325
|
"workflow.total_models": groups.reduce((n, g) => n + g.models.length, 0),
|
|
@@ -23,6 +23,7 @@ export async function scoreEnsemble(options) {
|
|
|
23
23
|
return withSpan({
|
|
24
24
|
name: "neurolink.workflow.judge.score",
|
|
25
25
|
tracer: tracers.workflow,
|
|
26
|
+
observationType: "span",
|
|
26
27
|
attributes: {
|
|
27
28
|
"workflow.judges_count": options.judges.length,
|
|
28
29
|
"workflow.responses_count": options.responses.length,
|
|
@@ -23,6 +23,7 @@ export async function conditionResponse(options) {
|
|
|
23
23
|
return withSpan({
|
|
24
24
|
name: "neurolink.workflow.response.condition",
|
|
25
25
|
tracer: tracers.workflow,
|
|
26
|
+
observationType: "span",
|
|
26
27
|
attributes: {
|
|
27
28
|
"workflow.conditioning.enabled": Boolean(options.config?.useConfidence),
|
|
28
29
|
"workflow.conditioning.has_synthesis_model": Boolean(options.config?.synthesisModel),
|