@juspay/neurolink 9.72.0 → 9.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +51 -51
- package/dist/lib/types/livekit.d.ts +134 -0
- package/dist/lib/voice/livekit/brain.js +1 -1
- package/dist/lib/voice/livekit/config.d.ts +12 -1
- package/dist/lib/voice/livekit/config.js +54 -0
- package/dist/lib/voice/livekit/eventBridge.js +4 -4
- package/dist/lib/voice/livekit/index.d.ts +9 -2
- package/dist/lib/voice/livekit/index.js +9 -2
- package/dist/lib/voice/livekit/realtimeEventBridge.d.ts +14 -0
- package/dist/lib/voice/livekit/realtimeEventBridge.js +161 -0
- package/dist/lib/voice/livekit/realtimeMcpTools.d.ts +31 -0
- package/dist/lib/voice/livekit/realtimeMcpTools.js +194 -0
- package/dist/lib/voice/livekit/realtimeVoiceAgent.d.ts +26 -0
- package/dist/lib/voice/livekit/realtimeVoiceAgent.js +362 -0
- package/dist/lib/voice/livekit/roomContext.d.ts +23 -0
- package/dist/lib/voice/livekit/roomContext.js +57 -0
- package/dist/lib/voice/livekit/roomDispatch.d.ts +24 -0
- package/dist/lib/voice/livekit/roomDispatch.js +31 -0
- package/dist/lib/voice/livekit/schemaSanitizer.d.ts +26 -0
- package/dist/lib/voice/livekit/schemaSanitizer.js +144 -0
- package/dist/lib/voice/livekit/vertexAuth.d.ts +30 -0
- package/dist/lib/voice/livekit/vertexAuth.js +73 -0
- package/dist/lib/voice/livekit/voiceAgent.js +47 -37
- package/dist/lib/voice/livekit/voiceAgentWorker.d.ts +2 -0
- package/dist/lib/voice/livekit/voiceAgentWorker.js +64 -0
- package/dist/types/livekit.d.ts +134 -0
- package/dist/voice/livekit/brain.js +1 -1
- package/dist/voice/livekit/config.d.ts +12 -1
- package/dist/voice/livekit/config.js +54 -0
- package/dist/voice/livekit/eventBridge.js +4 -4
- package/dist/voice/livekit/index.d.ts +9 -2
- package/dist/voice/livekit/index.js +9 -2
- package/dist/voice/livekit/realtimeEventBridge.d.ts +14 -0
- package/dist/voice/livekit/realtimeEventBridge.js +160 -0
- package/dist/voice/livekit/realtimeMcpTools.d.ts +31 -0
- package/dist/voice/livekit/realtimeMcpTools.js +193 -0
- package/dist/voice/livekit/realtimeVoiceAgent.d.ts +26 -0
- package/dist/voice/livekit/realtimeVoiceAgent.js +361 -0
- package/dist/voice/livekit/roomContext.d.ts +23 -0
- package/dist/voice/livekit/roomContext.js +56 -0
- package/dist/voice/livekit/roomDispatch.d.ts +24 -0
- package/dist/voice/livekit/roomDispatch.js +30 -0
- package/dist/voice/livekit/schemaSanitizer.d.ts +26 -0
- package/dist/voice/livekit/schemaSanitizer.js +143 -0
- package/dist/voice/livekit/vertexAuth.d.ts +30 -0
- package/dist/voice/livekit/vertexAuth.js +72 -0
- package/dist/voice/livekit/voiceAgent.js +47 -37
- package/dist/voice/livekit/voiceAgentWorker.d.ts +2 -0
- package/dist/voice/livekit/voiceAgentWorker.js +64 -0
- package/package.json +2 -1
package/dist/types/livekit.d.ts
CHANGED
|
@@ -155,6 +155,7 @@ export type LiveKitVoiceAgentConfig = {
|
|
|
155
155
|
conversationIdPrefix?: string;
|
|
156
156
|
/** Optional user id recorded alongside memory. */
|
|
157
157
|
userId?: string;
|
|
158
|
+
greeting?: string;
|
|
158
159
|
/** Silero VAD tuning (stricter = ignores background noise). */
|
|
159
160
|
vad?: LiveKitVadConfig;
|
|
160
161
|
/** Turn-detection tuning (VAD vs STT endpointing, delays). */
|
|
@@ -367,3 +368,136 @@ export type LiveKitEventBridgeHandle = {
|
|
|
367
368
|
/** Remove all listeners and stop publishing. Idempotent. */
|
|
368
369
|
dispose: () => void;
|
|
369
370
|
};
|
|
371
|
+
/**
|
|
372
|
+
* Credentials for LiveKit server-side REST calls (room create, agent dispatch).
|
|
373
|
+
* `url` accepts `ws(s)://` or `http(s)://`; helpers convert it to https.
|
|
374
|
+
*/
|
|
375
|
+
export type LiveKitServerCredentials = {
|
|
376
|
+
url: string;
|
|
377
|
+
apiKey: string;
|
|
378
|
+
apiSecret: string;
|
|
379
|
+
};
|
|
380
|
+
/**
|
|
381
|
+
* Realtime voice configuration resolved from the environment.
|
|
382
|
+
*
|
|
383
|
+
* In speech-to-speech mode one realtime model (Gemini Live on Vertex) does STT,
|
|
384
|
+
* reasoning, TTS, and turn detection — so there is no separate STT/TTS/VAD/EOU
|
|
385
|
+
* config. `resolveRealtimeVoiceConfig` fills every field from `process.env`
|
|
386
|
+
* (with defaults); `RealtimeVoiceAgentConfig` lets a caller override any of them.
|
|
387
|
+
*/
|
|
388
|
+
export type RealtimeVoiceConfig = {
|
|
389
|
+
/** Vertex project id (from VERTEX_PROJECT / GOOGLE_AUTH_* / GOOGLE_CLOUD_PROJECT_ID). */
|
|
390
|
+
project: string | undefined;
|
|
391
|
+
/** Vertex location; native-audio Live is served on `global`, not regionally. */
|
|
392
|
+
location: string;
|
|
393
|
+
/** Realtime model id (e.g. "gemini-live-2.5-flash"). */
|
|
394
|
+
model: string;
|
|
395
|
+
/** Optional Gemini voice name; omit for the plugin default. */
|
|
396
|
+
voice: string | undefined;
|
|
397
|
+
/** Response modality: "AUDIO" (native S2S) or "TEXT" (half-cascade). */
|
|
398
|
+
responseModality: string;
|
|
399
|
+
/** System prompt / instructions for the agent. */
|
|
400
|
+
systemPrompt: string;
|
|
401
|
+
/** Opening line the agent speaks on connect ("" disables). */
|
|
402
|
+
greeting: string;
|
|
403
|
+
/** Whether to bridge Lighthouse MCP tools as Gemini function tools. */
|
|
404
|
+
toolsEnabled: boolean;
|
|
405
|
+
/** Full URL of the MCP server the tools are bridged from. */
|
|
406
|
+
mcpUrl: string;
|
|
407
|
+
/** Grace period after the caller leaves before the job shuts down (ms). */
|
|
408
|
+
emptyRoomGraceMs: number;
|
|
409
|
+
/** Deadline for a participant to join before the job shuts down (ms). */
|
|
410
|
+
joinDeadlineMs: number;
|
|
411
|
+
/** How long a HITL confirmation waits before being treated as a decline (ms). */
|
|
412
|
+
hitlTimeoutMs: number;
|
|
413
|
+
/** Interval for the RSS/heap metrics log (ms). */
|
|
414
|
+
metricsIntervalMs: number;
|
|
415
|
+
};
|
|
416
|
+
/** A single log record handed to a `RealtimeVoiceAgentConfig.onLog` sink. */
|
|
417
|
+
export type RealtimeVoiceLogEntry = {
|
|
418
|
+
level: "debug" | "info" | "warn" | "error";
|
|
419
|
+
message: string;
|
|
420
|
+
timestamp: number;
|
|
421
|
+
data?: unknown;
|
|
422
|
+
};
|
|
423
|
+
export type RealtimeVoiceLogContext = {
|
|
424
|
+
room: string;
|
|
425
|
+
};
|
|
426
|
+
/**
|
|
427
|
+
* Options for `defineRealtimeVoiceAgent`. Every field is optional: omitted
|
|
428
|
+
* values fall back to `resolveRealtimeVoiceConfig()` (i.e. the environment), so
|
|
429
|
+
* a caller can use `defineRealtimeVoiceAgent()` with no arguments and configure
|
|
430
|
+
* everything via env.
|
|
431
|
+
*/
|
|
432
|
+
export type RealtimeVoiceAgentConfig = {
|
|
433
|
+
project?: string;
|
|
434
|
+
location?: string;
|
|
435
|
+
model?: string;
|
|
436
|
+
voice?: string;
|
|
437
|
+
responseModality?: string;
|
|
438
|
+
systemPrompt?: string;
|
|
439
|
+
greeting?: string;
|
|
440
|
+
/** MCP tool bridging overrides. */
|
|
441
|
+
tools?: {
|
|
442
|
+
enabled?: boolean;
|
|
443
|
+
mcpUrl?: string;
|
|
444
|
+
};
|
|
445
|
+
/** Data-channel topic for outbound events (default "ai-events"). */
|
|
446
|
+
eventsTopic?: string;
|
|
447
|
+
/** Data-channel topic for inbound control messages (default "ai-control"). */
|
|
448
|
+
controlTopic?: string;
|
|
449
|
+
/**
|
|
450
|
+
* Optional sink for the agent's own logs. When set, the realtime agent wires
|
|
451
|
+
* NeuroLink's logger to this callback for the duration of the call, so a host
|
|
452
|
+
* can forward worker logs into its logging pipeline. Each record is tagged
|
|
453
|
+
* with per-call context (the room name). Subject to the logger's level gate:
|
|
454
|
+
* without debug mode only `error` records are emitted (set `NEUROLINK_DEBUG`).
|
|
455
|
+
*/
|
|
456
|
+
onLog?: (entry: RealtimeVoiceLogEntry, ctx: RealtimeVoiceLogContext) => void;
|
|
457
|
+
};
|
|
458
|
+
/** Auth token + base64 MCP execution context decoded from a room's metadata. */
|
|
459
|
+
export type LiveKitRoomCallContext = {
|
|
460
|
+
/** Lighthouse access JWT used as `x-auth-token` to the MCP server. */
|
|
461
|
+
authToken: string;
|
|
462
|
+
/** base64(JSON) MCP execution context used as `x-context` (or "" if absent). */
|
|
463
|
+
xContext: string;
|
|
464
|
+
};
|
|
465
|
+
/** Publishes a single voice event envelope onto the room data channel. */
|
|
466
|
+
export type RealtimeEventPublisher = (type: LiveKitVoiceEventType, data: Record<string, unknown>) => void;
|
|
467
|
+
/** Requests a HITL confirmation and resolves to the user's decision. */
|
|
468
|
+
export type RealtimeConfirmationRequester = (toolName: string, args: Record<string, unknown>) => Promise<boolean>;
|
|
469
|
+
/** Handle returned by `attachRealtimeEventBridge`. */
|
|
470
|
+
export type RealtimeEventBridgeHandle = {
|
|
471
|
+
/** Publish an outbound event to the browser (data packet or text stream). */
|
|
472
|
+
publishEvent: RealtimeEventPublisher;
|
|
473
|
+
/** Open a HITL prompt and await the browser's decision (timeout = decline). */
|
|
474
|
+
requestConfirmation: RealtimeConfirmationRequester;
|
|
475
|
+
/** Remove the control-channel listener and clear pending confirmations. */
|
|
476
|
+
dispose: () => void;
|
|
477
|
+
};
|
|
478
|
+
/** Inputs to `attachRealtimeEventBridge`. */
|
|
479
|
+
export type RealtimeEventBridgeParams = {
|
|
480
|
+
/** The LiveKit room for this call (from the job context). */
|
|
481
|
+
room: LiveKitBridgeRoom;
|
|
482
|
+
/** HITL confirmation timeout in ms before a request is auto-declined. */
|
|
483
|
+
hitlTimeoutMs?: number;
|
|
484
|
+
/** Outbound events topic (default "ai-events"). */
|
|
485
|
+
eventsTopic?: string;
|
|
486
|
+
/** Inbound control topic (default "ai-control"). */
|
|
487
|
+
controlTopic?: string;
|
|
488
|
+
/** Payloads larger than this are sent via the chunked text stream (default 12000). */
|
|
489
|
+
maxInlineBytes?: number;
|
|
490
|
+
};
|
|
491
|
+
/** Inputs to `buildRealtimeMcpTools`. */
|
|
492
|
+
export type BuildRealtimeMcpToolsParams = {
|
|
493
|
+
/** Full URL of the MCP server (e.g. ".../ai/mcp/v2"). */
|
|
494
|
+
mcpUrl: string;
|
|
495
|
+
/** Lighthouse access JWT forwarded as `x-auth-token`. */
|
|
496
|
+
authToken: string;
|
|
497
|
+
/** base64(JSON) execution context forwarded as `x-context`. */
|
|
498
|
+
xContext: string;
|
|
499
|
+
/** Publishes tool start/result events to the browser. */
|
|
500
|
+
publishEvent: RealtimeEventPublisher;
|
|
501
|
+
/** Opens a HITL confirmation for destructive tools and awaits the decision. */
|
|
502
|
+
requestConfirmation: RealtimeConfirmationRequester;
|
|
503
|
+
};
|
|
@@ -61,7 +61,7 @@ export function createVoiceBrain(config) {
|
|
|
61
61
|
});
|
|
62
62
|
for await (const chunk of result.stream) {
|
|
63
63
|
if (signal?.aborted) {
|
|
64
|
-
logger.debug("
|
|
64
|
+
logger.debug("voice.brain.turnAborted", { reason: "signal-aborted" });
|
|
65
65
|
return;
|
|
66
66
|
}
|
|
67
67
|
const delta = extractTextDelta(chunk);
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* See docs/features/livekit-voice-agent.md.
|
|
9
9
|
*/
|
|
10
|
-
import type { LiveKitServerConfig, LiveKitBrainDefaults } from "../../types/index.js";
|
|
10
|
+
import type { LiveKitServerConfig, LiveKitBrainDefaults, RealtimeVoiceConfig } from "../../types/index.js";
|
|
11
11
|
/**
|
|
12
12
|
* Resolve LiveKit server connection settings from the environment.
|
|
13
13
|
*
|
|
@@ -23,6 +23,17 @@ export declare function resolveLiveKitServerConfig(): LiveKitServerConfig;
|
|
|
23
23
|
* `VOICE_LLM_MODEL`.
|
|
24
24
|
*/
|
|
25
25
|
export declare function resolveBrainDefaults(): LiveKitBrainDefaults;
|
|
26
|
+
/**
|
|
27
|
+
* Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
|
|
28
|
+
* the environment.
|
|
29
|
+
*
|
|
30
|
+
* Every value has a safe default so a worker can run with no realtime-specific
|
|
31
|
+
* env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
|
|
32
|
+
* service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
|
|
33
|
+
* `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
|
|
34
|
+
* `${LIGHTHOUSE_URL}/ai/mcp/v2`.
|
|
35
|
+
*/
|
|
36
|
+
export declare function resolveRealtimeVoiceConfig(): RealtimeVoiceConfig;
|
|
26
37
|
/**
|
|
27
38
|
* Resolve the semantic end-of-utterance (EOU) turn-detection settings.
|
|
28
39
|
*
|
|
@@ -25,6 +25,22 @@ function readEnv(name, fallback) {
|
|
|
25
25
|
}
|
|
26
26
|
return fallback;
|
|
27
27
|
}
|
|
28
|
+
function readOptionalEnv(name) {
|
|
29
|
+
const value = process.env[name];
|
|
30
|
+
return typeof value === "string" && value.trim().length > 0
|
|
31
|
+
? value.trim()
|
|
32
|
+
: undefined;
|
|
33
|
+
}
|
|
34
|
+
function readNumberEnv(name, fallback) {
|
|
35
|
+
const raw = process.env[name];
|
|
36
|
+
if (typeof raw === "string" && raw.trim().length > 0) {
|
|
37
|
+
const parsed = Number(raw.trim());
|
|
38
|
+
if (Number.isFinite(parsed) && parsed > 0) {
|
|
39
|
+
return parsed;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return fallback;
|
|
43
|
+
}
|
|
28
44
|
/**
|
|
29
45
|
* Resolve LiveKit server connection settings from the environment.
|
|
30
46
|
*
|
|
@@ -51,6 +67,44 @@ export function resolveBrainDefaults() {
|
|
|
51
67
|
model: readEnv("VOICE_LLM_MODEL", DEFAULT_LLM_MODEL),
|
|
52
68
|
};
|
|
53
69
|
}
|
|
70
|
+
const DEFAULT_REALTIME_MODEL = "gemini-live-2.5-flash";
|
|
71
|
+
const DEFAULT_REALTIME_LOCATION = "global";
|
|
72
|
+
const DEFAULT_RESPONSE_MODALITY = "AUDIO";
|
|
73
|
+
const DEFAULT_SYSTEM_PROMPT = "You are a concise voice assistant for a merchant dashboard. Keep replies short and spoken.";
|
|
74
|
+
const DEFAULT_GREETING = "Briefly greet the merchant and ask how you can help with their store today.";
|
|
75
|
+
const DEFAULT_LIGHTHOUSE_URL = "http://localhost:5173";
|
|
76
|
+
const DEFAULT_MCP_PATH = "/ai/mcp/v2";
|
|
77
|
+
/**
|
|
78
|
+
* Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
|
|
79
|
+
* the environment.
|
|
80
|
+
*
|
|
81
|
+
* Every value has a safe default so a worker can run with no realtime-specific
|
|
82
|
+
* env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
|
|
83
|
+
* service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
|
|
84
|
+
* `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
|
|
85
|
+
* `${LIGHTHOUSE_URL}/ai/mcp/v2`.
|
|
86
|
+
*/
|
|
87
|
+
export function resolveRealtimeVoiceConfig() {
|
|
88
|
+
const lighthouseUrl = readEnv("LIGHTHOUSE_URL", DEFAULT_LIGHTHOUSE_URL);
|
|
89
|
+
const mcpUrl = readEnv("VOICE_MCP_URL", `${lighthouseUrl}${DEFAULT_MCP_PATH}`);
|
|
90
|
+
return {
|
|
91
|
+
project: readOptionalEnv("VERTEX_PROJECT") ??
|
|
92
|
+
readOptionalEnv("GOOGLE_AUTH_BREEZE_PROJECT_ID") ??
|
|
93
|
+
readOptionalEnv("GOOGLE_CLOUD_PROJECT_ID"),
|
|
94
|
+
location: readEnv("VERTEX_LOCATION", DEFAULT_REALTIME_LOCATION),
|
|
95
|
+
model: readEnv("VOICE_LIVE_MODEL", DEFAULT_REALTIME_MODEL),
|
|
96
|
+
voice: readOptionalEnv("VOICE_S2S_VOICE"),
|
|
97
|
+
responseModality: readEnv("VOICE_RESPONSE_MODALITY", DEFAULT_RESPONSE_MODALITY).toUpperCase(),
|
|
98
|
+
systemPrompt: readEnv("VOICE_SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT),
|
|
99
|
+
greeting: readEnv("VOICE_GREETING", DEFAULT_GREETING),
|
|
100
|
+
toolsEnabled: readEnv("VOICE_S2S_TOOLS", "").toLowerCase() === "true",
|
|
101
|
+
mcpUrl,
|
|
102
|
+
emptyRoomGraceMs: readNumberEnv("VOICE_EMPTY_ROOM_GRACE_MS", 30_000),
|
|
103
|
+
joinDeadlineMs: readNumberEnv("VOICE_JOIN_DEADLINE_MS", 60_000),
|
|
104
|
+
hitlTimeoutMs: readNumberEnv("VOICE_HITL_TIMEOUT_MS", 45_000),
|
|
105
|
+
metricsIntervalMs: readNumberEnv("VOICE_METRICS_INTERVAL_MS", 10_000),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
54
108
|
const EOU_TRUTHY = new Set(["1", "true", "english", "en", "on", "yes"]);
|
|
55
109
|
/**
|
|
56
110
|
* Resolve the semantic end-of-utterance (EOU) turn-detection settings.
|
|
@@ -191,7 +191,7 @@ export async function attachEventBridge(params) {
|
|
|
191
191
|
const json = JSON.stringify(envelope);
|
|
192
192
|
const bytes = encoder.encode(json);
|
|
193
193
|
const onError = (transport) => (error) => {
|
|
194
|
-
logger.warn("
|
|
194
|
+
logger.warn("voice.bridge.publishFailed", {
|
|
195
195
|
transport,
|
|
196
196
|
type: event.type,
|
|
197
197
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -304,7 +304,7 @@ export async function attachEventBridge(params) {
|
|
|
304
304
|
parsed = JSON.parse(decoder.decode(payload));
|
|
305
305
|
}
|
|
306
306
|
catch (error) {
|
|
307
|
-
logger.warn("
|
|
307
|
+
logger.warn("voice.bridge.controlMalformed", {
|
|
308
308
|
error: error instanceof Error ? error.message : String(error),
|
|
309
309
|
});
|
|
310
310
|
return;
|
|
@@ -332,7 +332,7 @@ export async function attachEventBridge(params) {
|
|
|
332
332
|
};
|
|
333
333
|
room.on(RoomEvent.DataReceived, onData);
|
|
334
334
|
publish({ type: "status", data: { state: "listening" } });
|
|
335
|
-
logger.info("
|
|
335
|
+
logger.info("voice.bridge.attached", {
|
|
336
336
|
eventsTopic,
|
|
337
337
|
controlTopic,
|
|
338
338
|
filtered: include !== undefined,
|
|
@@ -353,7 +353,7 @@ export async function attachEventBridge(params) {
|
|
|
353
353
|
emitter.off(STREAM_END_EVENT, onStreamComplete);
|
|
354
354
|
emitter.off(STREAM_ERROR_EVENT, onStreamError);
|
|
355
355
|
room.off(RoomEvent.DataReceived, onData);
|
|
356
|
-
logger.info("
|
|
356
|
+
logger.info("voice.bridge.disposed", {});
|
|
357
357
|
},
|
|
358
358
|
};
|
|
359
359
|
}
|
|
@@ -8,8 +8,15 @@
|
|
|
8
8
|
* See docs/features/livekit-voice-agent.md.
|
|
9
9
|
*/
|
|
10
10
|
export { createVoiceBrain } from "./brain.js";
|
|
11
|
-
export { resolveLiveKitServerConfig, resolveBrainDefaults } from "./config.js";
|
|
11
|
+
export { resolveLiveKitServerConfig, resolveBrainDefaults, resolveRealtimeVoiceConfig, } from "./config.js";
|
|
12
12
|
export { attachEventBridge } from "./eventBridge.js";
|
|
13
|
+
export { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
|
|
13
14
|
export { mintJoinToken } from "./tokens.js";
|
|
15
|
+
export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
|
|
14
16
|
export { defineVoiceAgent } from "./voiceAgent.js";
|
|
15
|
-
export {
|
|
17
|
+
export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
|
|
18
|
+
export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
|
|
19
|
+
export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
|
|
20
|
+
export { readCallContextFromRoom } from "./roomContext.js";
|
|
21
|
+
export { sanitizeSchema, sanitizeToolParameters, findSchemaIssue, } from "./schemaSanitizer.js";
|
|
22
|
+
export { buildRealtimeMcpTools, mcpResultToText } from "./realtimeMcpTools.js";
|
|
@@ -8,8 +8,15 @@
|
|
|
8
8
|
* See docs/features/livekit-voice-agent.md.
|
|
9
9
|
*/
|
|
10
10
|
export { createVoiceBrain } from "./brain.js";
|
|
11
|
-
export { resolveLiveKitServerConfig, resolveBrainDefaults } from "./config.js";
|
|
11
|
+
export { resolveLiveKitServerConfig, resolveBrainDefaults, resolveRealtimeVoiceConfig, } from "./config.js";
|
|
12
12
|
export { attachEventBridge } from "./eventBridge.js";
|
|
13
|
+
export { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
|
|
13
14
|
export { mintJoinToken } from "./tokens.js";
|
|
15
|
+
export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
|
|
14
16
|
export { defineVoiceAgent } from "./voiceAgent.js";
|
|
15
|
-
export {
|
|
17
|
+
export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
|
|
18
|
+
export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
|
|
19
|
+
export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
|
|
20
|
+
export { readCallContextFromRoom } from "./roomContext.js";
|
|
21
|
+
export { sanitizeSchema, sanitizeToolParameters, findSchemaIssue, } from "./schemaSanitizer.js";
|
|
22
|
+
export { buildRealtimeMcpTools, mcpResultToText } from "./realtimeMcpTools.js";
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Realtime data-channel event bridge.
|
|
3
|
+
*
|
|
4
|
+
* The cascaded bridge (`eventBridge.ts`) is driven by NeuroLink's event emitter.
|
|
5
|
+
*/
|
|
6
|
+
import type { RealtimeEventBridgeHandle, RealtimeEventBridgeParams } from "../../types/index.js";
|
|
7
|
+
/**
|
|
8
|
+
* Attach the realtime event bridge to a room.
|
|
9
|
+
*
|
|
10
|
+
* Returns `publishEvent` (worker → browser), `requestConfirmation` (HITL
|
|
11
|
+
* round-trip), and `dispose` (remove the control listener, decline anything
|
|
12
|
+
* still pending).
|
|
13
|
+
*/
|
|
14
|
+
export declare function attachRealtimeEventBridge(params: RealtimeEventBridgeParams): Promise<RealtimeEventBridgeHandle>;
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Realtime data-channel event bridge.
|
|
3
|
+
*
|
|
4
|
+
* The cascaded bridge (`eventBridge.ts`) is driven by NeuroLink's event emitter.
|
|
5
|
+
*/
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import { logger } from "../../utils/logger.js";
|
|
8
|
+
const DEFAULT_EVENTS_TOPIC = "ai-events";
|
|
9
|
+
const DEFAULT_CONTROL_TOPIC = "ai-control";
|
|
10
|
+
const DEFAULT_MAX_INLINE_BYTES = 12_000;
|
|
11
|
+
const DEFAULT_HITL_TIMEOUT_MS = 45_000;
|
|
12
|
+
const hitlControlMessageSchema = z.object({
|
|
13
|
+
action: z.enum(["hitl:accept", "hitl:reject"]),
|
|
14
|
+
confirmationId: z.string(),
|
|
15
|
+
});
|
|
16
|
+
/**
|
|
17
|
+
* Attach the realtime event bridge to a room.
|
|
18
|
+
*
|
|
19
|
+
* Returns `publishEvent` (worker → browser), `requestConfirmation` (HITL
|
|
20
|
+
* round-trip), and `dispose` (remove the control listener, decline anything
|
|
21
|
+
* still pending).
|
|
22
|
+
*/
|
|
23
|
+
export async function attachRealtimeEventBridge(params) {
|
|
24
|
+
const { room } = params;
|
|
25
|
+
const eventsTopic = params.eventsTopic ?? DEFAULT_EVENTS_TOPIC;
|
|
26
|
+
const controlTopic = params.controlTopic ?? DEFAULT_CONTROL_TOPIC;
|
|
27
|
+
const maxInlineBytes = params.maxInlineBytes ?? DEFAULT_MAX_INLINE_BYTES;
|
|
28
|
+
const hitlTimeoutMs = params.hitlTimeoutMs ?? DEFAULT_HITL_TIMEOUT_MS;
|
|
29
|
+
const { RoomEvent } = await import("@livekit/rtc-node");
|
|
30
|
+
const encoder = new TextEncoder();
|
|
31
|
+
const decoder = new TextDecoder();
|
|
32
|
+
let seq = 0;
|
|
33
|
+
let disposed = false;
|
|
34
|
+
const startedAt = Date.now();
|
|
35
|
+
const publishEvent = (type, data) => {
|
|
36
|
+
if (disposed) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const participant = room.localParticipant;
|
|
40
|
+
if (!participant) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
seq += 1;
|
|
45
|
+
const json = JSON.stringify({ seq, ts: Date.now(), type, data });
|
|
46
|
+
const bytes = encoder.encode(json);
|
|
47
|
+
const via = bytes.byteLength <= maxInlineBytes ? "data" : "stream";
|
|
48
|
+
logger.debug("realtime.bridge.publish", {
|
|
49
|
+
ms: Date.now() - startedAt,
|
|
50
|
+
seq,
|
|
51
|
+
type,
|
|
52
|
+
via,
|
|
53
|
+
bytes: bytes.byteLength,
|
|
54
|
+
});
|
|
55
|
+
if (via === "data") {
|
|
56
|
+
void participant.publishData(bytes, {
|
|
57
|
+
reliable: true,
|
|
58
|
+
topic: eventsTopic,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
void participant.sendText(json, { topic: eventsTopic });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
/* non-fatal — the UI bridge is best-effort */
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
// HITL: WRITE-labeled tools pause for user confirmation. We publish a
|
|
70
|
+
// `hitl-prompt`, the UI replies on the control topic, and we resolve the
|
|
71
|
+
// pending request by confirmationId. A timeout is treated as a decline so a
|
|
72
|
+
// turn can never hang waiting on the user.
|
|
73
|
+
const pendingHitl = new Map();
|
|
74
|
+
let hitlSeq = 0;
|
|
75
|
+
const onData = (...args) => {
|
|
76
|
+
const payload = args[0];
|
|
77
|
+
const topic = args[3];
|
|
78
|
+
if (topic !== controlTopic || !(payload instanceof Uint8Array)) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
let raw;
|
|
82
|
+
try {
|
|
83
|
+
raw = JSON.parse(decoder.decode(payload));
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
const message = hitlControlMessageSchema.safeParse(raw);
|
|
89
|
+
if (!message.success) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
const { action, confirmationId } = message.data;
|
|
93
|
+
const resolver = pendingHitl.get(confirmationId);
|
|
94
|
+
if (resolver) {
|
|
95
|
+
logger.debug("realtime.bridge.controlReceived", {
|
|
96
|
+
action,
|
|
97
|
+
confirmationId,
|
|
98
|
+
});
|
|
99
|
+
pendingHitl.delete(confirmationId);
|
|
100
|
+
resolver(action === "hitl:accept");
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
logger.warn("realtime.bridge.controlUnmatched", {
|
|
104
|
+
action,
|
|
105
|
+
confirmationId,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
room.on(RoomEvent.DataReceived, onData);
|
|
110
|
+
const requestConfirmation = (toolName, args) => {
|
|
111
|
+
if (disposed) {
|
|
112
|
+
logger.warn("realtime.bridge.hitlDisposed", { toolName });
|
|
113
|
+
return Promise.resolve(false);
|
|
114
|
+
}
|
|
115
|
+
hitlSeq += 1;
|
|
116
|
+
const confirmationId = `hitl_${seq}_${hitlSeq}`;
|
|
117
|
+
publishEvent("hitl-prompt", {
|
|
118
|
+
confirmationId,
|
|
119
|
+
toolName,
|
|
120
|
+
actionType: `Run ${toolName}`,
|
|
121
|
+
arguments: args ?? {},
|
|
122
|
+
});
|
|
123
|
+
logger.info("realtime.bridge.hitlPrompt", { toolName, confirmationId });
|
|
124
|
+
return new Promise((resolve) => {
|
|
125
|
+
const timer = setTimeout(() => {
|
|
126
|
+
pendingHitl.delete(confirmationId);
|
|
127
|
+
logger.warn("realtime.bridge.hitlTimeout", {
|
|
128
|
+
toolName,
|
|
129
|
+
confirmationId,
|
|
130
|
+
timeoutMs: hitlTimeoutMs,
|
|
131
|
+
});
|
|
132
|
+
resolve(false);
|
|
133
|
+
}, hitlTimeoutMs);
|
|
134
|
+
pendingHitl.set(confirmationId, (approved) => {
|
|
135
|
+
clearTimeout(timer);
|
|
136
|
+
logger.info("realtime.bridge.hitlResolved", {
|
|
137
|
+
toolName,
|
|
138
|
+
confirmationId,
|
|
139
|
+
approved,
|
|
140
|
+
});
|
|
141
|
+
resolve(approved);
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
};
|
|
145
|
+
return {
|
|
146
|
+
publishEvent,
|
|
147
|
+
requestConfirmation,
|
|
148
|
+
dispose() {
|
|
149
|
+
if (disposed) {
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
disposed = true;
|
|
153
|
+
room.off(RoomEvent.DataReceived, onData);
|
|
154
|
+
for (const [, resolver] of pendingHitl) {
|
|
155
|
+
resolver(false);
|
|
156
|
+
}
|
|
157
|
+
pendingHitl.clear();
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCP tools → Gemini function tools bridge (realtime mode).
|
|
3
|
+
*
|
|
4
|
+
* In speech-to-speech mode Gemini owns the tool loop, so an external MCP
|
|
5
|
+
* server's tools are registered as LiveKit/Gemini function tools rather than
|
|
6
|
+
* NeuroLink tools. This connects to the MCP server as the calling user
|
|
7
|
+
* (`x-auth-token` + base64 `x-context`), sanitizes each tool's schema into the
|
|
8
|
+
* Gemini-safe subset, wires WRITE-labeled (destructive) tools through a HITL
|
|
9
|
+
* confirmation, and forwards tool start/result events to the browser bridge.
|
|
10
|
+
*
|
|
11
|
+
* `@livekit/agents` and `@modelcontextprotocol/sdk` are imported dynamically so
|
|
12
|
+
* the core package does not require them unless the realtime agent is used.
|
|
13
|
+
*
|
|
14
|
+
* See docs/features/livekit-voice-agent.md.
|
|
15
|
+
*/
|
|
16
|
+
import type { llm as llmNs } from "@livekit/agents";
|
|
17
|
+
import type { BuildRealtimeMcpToolsParams } from "../../types/index.js";
|
|
18
|
+
/** Flatten an MCP tool result's text content into a single string for the model. */
|
|
19
|
+
export declare function mcpResultToText(result: unknown): string;
|
|
20
|
+
/**
|
|
21
|
+
* Connect to the MCP server as the calling user and bridge every tool into a
|
|
22
|
+
* LiveKit `ToolContext`. Returns the tool map + the client (close on shutdown).
|
|
23
|
+
* The server already scopes the tool list by `x-context`, so no client-side
|
|
24
|
+
* filtering is needed.
|
|
25
|
+
*/
|
|
26
|
+
export declare function buildRealtimeMcpTools(params: BuildRealtimeMcpToolsParams): Promise<{
|
|
27
|
+
tools: llmNs.ToolContext;
|
|
28
|
+
client: {
|
|
29
|
+
close: () => Promise<void>;
|
|
30
|
+
};
|
|
31
|
+
}>;
|