@juspay/neurolink 12.49.20 → 12.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/browser/neurolink.min.js +371 -371
- package/dist/core/conversationMemoryManager.js +10 -4
- package/dist/core/redisConversationMemoryManager.js +6 -5
- package/dist/neurolink.d.ts +52 -1
- package/dist/neurolink.js +195 -0
- package/dist/types/conversation.d.ts +109 -0
- package/dist/types/livekit.d.ts +246 -3
- package/dist/types/sdk.d.ts +1 -1
- package/dist/types/stream.d.ts +8 -1
- package/dist/utils/conversationTurnIdentity.d.ts +24 -0
- package/dist/utils/conversationTurnIdentity.js +81 -0
- package/dist/voice/livekit/brain.js +10 -7
- package/dist/voice/livekit/index.d.ts +2 -0
- package/dist/voice/livekit/index.js +2 -0
- package/dist/voice/livekit/memoryBinding.d.ts +47 -0
- package/dist/voice/livekit/memoryBinding.js +152 -0
- package/dist/voice/livekit/realtimeMcpTools.js +20 -1
- package/dist/voice/livekit/realtimeMemory.d.ts +22 -0
- package/dist/voice/livekit/realtimeMemory.js +111 -0
- package/dist/voice/livekit/realtimeVoiceAgent.js +40 -1
- package/dist/voice/livekit/turnRecorder.d.ts +38 -0
- package/dist/voice/livekit/turnRecorder.js +362 -0
- package/dist/voice/livekit/voiceAgent.js +43 -5
- package/docs-site/static/search-index.json +1 -0
- package/package.json +2 -1
package/dist/types/livekit.d.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import type { NeuroLinkEvents, TypedEventEmitter } from "./common.js";
|
|
13
13
|
import type { StreamOptions, StreamResult } from "./stream.js";
|
|
14
|
+
import type { ConversationSeedMessage, ConversationSeedOptions, RecordConversationTurnOptions, RecordConversationTurnResult, StreamEventSequence } from "./conversation.js";
|
|
14
15
|
/**
|
|
15
16
|
* Minimal structural shape of the NeuroLink instance the brain depends on.
|
|
16
17
|
*
|
|
@@ -57,6 +58,18 @@ export type LiveKitBrainTurn = {
|
|
|
57
58
|
conversationId: string;
|
|
58
59
|
/** Cancellation signal; aborting stops the in-flight LLM and tool calls. */
|
|
59
60
|
signal?: AbortSignal;
|
|
61
|
+
/**
|
|
62
|
+
* User the conversation belongs to for THIS call, overriding the brain's
|
|
63
|
+
* static `userId`. A call whose memory binding verified passes the bound
|
|
64
|
+
* user here so the turn is stored under the person on the call.
|
|
65
|
+
*/
|
|
66
|
+
userId?: string;
|
|
67
|
+
/**
|
|
68
|
+
* Run this turn without reading or writing conversation memory. Used for the
|
|
69
|
+
* agent's opening greeting on a bound call, which is an instruction to the
|
|
70
|
+
* agent and must not land in the person's conversation as something they said.
|
|
71
|
+
*/
|
|
72
|
+
ephemeral?: boolean;
|
|
60
73
|
};
|
|
61
74
|
/**
|
|
62
75
|
* The brain's public surface: stream the assistant reply as text deltas.
|
|
@@ -155,6 +168,15 @@ export type LiveKitVoiceAgentConfig = {
|
|
|
155
168
|
conversationIdPrefix?: string;
|
|
156
169
|
/** Optional user id recorded alongside memory. */
|
|
157
170
|
userId?: string;
|
|
171
|
+
/**
|
|
172
|
+
* Per-call memory identity. When set, the call's room metadata is checked
|
|
173
|
+
* for a signed memory binding (see `signVoiceMemoryBinding`); a binding that
|
|
174
|
+
* verifies replaces `userId` and the room-derived conversation id for that
|
|
175
|
+
* call, and the turns the brain streams are stamped as voice. A call whose
|
|
176
|
+
* binding is missing or does not verify runs with the static `userId` and
|
|
177
|
+
* the room-derived conversation id exactly as before.
|
|
178
|
+
*/
|
|
179
|
+
memory?: LiveKitVoiceAgentMemoryConfig;
|
|
158
180
|
greeting?: string;
|
|
159
181
|
/** Silero VAD tuning (stricter = ignores background noise). */
|
|
160
182
|
vad?: LiveKitVadConfig;
|
|
@@ -170,6 +192,11 @@ export type LiveKitVoiceAgentConfig = {
|
|
|
170
192
|
*/
|
|
171
193
|
events?: LiveKitEventBridgeConfig;
|
|
172
194
|
};
|
|
195
|
+
/** Per-call memory identity for the cascaded voice agent. */
|
|
196
|
+
export type LiveKitVoiceAgentMemoryConfig = {
|
|
197
|
+
/** Shared secret the binding is signed with. Defaults to `NEUROLINK_VOICE_MEMORY_SECRET`. */
|
|
198
|
+
secret?: string;
|
|
199
|
+
};
|
|
173
200
|
/** Options for `startVoiceAgentWorker` — launches the LiveKit Agents worker. */
|
|
174
201
|
export type LiveKitWorkerLaunchOptions = {
|
|
175
202
|
/**
|
|
@@ -208,7 +235,7 @@ export type LiveKitTokenRequest = {
|
|
|
208
235
|
ttlSeconds?: number;
|
|
209
236
|
};
|
|
210
237
|
/** Discriminant tags for outbound voice events. */
|
|
211
|
-
export type LiveKitVoiceEventType = "user-text" | "text" | "tool-start" | "tool-result" | "status" | "hitl-prompt" | "done";
|
|
238
|
+
export type LiveKitVoiceEventType = "user-text" | "text" | "tool-start" | "tool-result" | "status" | "hitl-prompt" | "done" | "memory";
|
|
212
239
|
/**
|
|
213
240
|
* A user STT transcript for display. Interim partials stream with
|
|
214
241
|
* `final: false`; the end-of-utterance result has `final: true`. The client
|
|
@@ -277,15 +304,32 @@ export type LiveKitVoiceHitlPromptEvent = {
|
|
|
277
304
|
allowModification?: boolean;
|
|
278
305
|
};
|
|
279
306
|
};
|
|
280
|
-
/**
|
|
307
|
+
/**
|
|
308
|
+
* The current turn has finished. `turnId` is set when the call's turns are
|
|
309
|
+
* being recorded: it is the id the stored messages carry (`${turnId}:user`,
|
|
310
|
+
* `${turnId}:assistant`), so a client can merge its live copy with history.
|
|
311
|
+
*/
|
|
281
312
|
export type LiveKitVoiceDoneEvent = {
|
|
282
313
|
type: "done";
|
|
283
314
|
data: {
|
|
284
315
|
reason?: string;
|
|
316
|
+
turnId?: string;
|
|
317
|
+
};
|
|
318
|
+
};
|
|
319
|
+
/**
|
|
320
|
+
* Something about how the call is being remembered. `not-saved` means a turn
|
|
321
|
+
* could not be stored after retries; the call carries on, and a client may
|
|
322
|
+
* show a quiet note on that turn.
|
|
323
|
+
*/
|
|
324
|
+
export type LiveKitVoiceMemoryEvent = {
|
|
325
|
+
type: "memory";
|
|
326
|
+
data: {
|
|
327
|
+
state: "not-saved";
|
|
328
|
+
turnId: string;
|
|
285
329
|
};
|
|
286
330
|
};
|
|
287
331
|
/** Discriminated union of all outbound voice events (before enveloping). */
|
|
288
|
-
export type LiveKitVoiceEvent = LiveKitVoiceUserTextEvent | LiveKitVoiceTextEvent | LiveKitVoiceToolStartEvent | LiveKitVoiceToolResultEvent | LiveKitVoiceStatusEvent | LiveKitVoiceHitlPromptEvent | LiveKitVoiceDoneEvent;
|
|
332
|
+
export type LiveKitVoiceEvent = LiveKitVoiceUserTextEvent | LiveKitVoiceTextEvent | LiveKitVoiceToolStartEvent | LiveKitVoiceToolResultEvent | LiveKitVoiceStatusEvent | LiveKitVoiceHitlPromptEvent | LiveKitVoiceDoneEvent | LiveKitVoiceMemoryEvent;
|
|
289
333
|
/**
|
|
290
334
|
* Wire format published to the browser: a `LiveKitVoiceEvent` plus a monotonic
|
|
291
335
|
* sequence number and a timestamp so the client can order and de-duplicate.
|
|
@@ -442,6 +486,14 @@ export type RealtimeVoiceAgentConfig = {
|
|
|
442
486
|
enabled?: boolean;
|
|
443
487
|
mcpUrl?: string;
|
|
444
488
|
};
|
|
489
|
+
/**
|
|
490
|
+
* Persist the call's turns to conversation memory and seed the model with
|
|
491
|
+
* the conversation's recent history. Off unless set. Takes effect only for
|
|
492
|
+
* a call whose room metadata carries a verified memory binding; otherwise
|
|
493
|
+
* the call runs exactly as it would without this option, and nothing is
|
|
494
|
+
* stored.
|
|
495
|
+
*/
|
|
496
|
+
memory?: RealtimeVoiceMemoryConfig;
|
|
445
497
|
/** Data-channel topic for outbound events (default "ai-events"). */
|
|
446
498
|
eventsTopic?: string;
|
|
447
499
|
/** Data-channel topic for inbound control messages (default "ai-control"). */
|
|
@@ -455,6 +507,186 @@ export type RealtimeVoiceAgentConfig = {
|
|
|
455
507
|
*/
|
|
456
508
|
onLog?: (entry: RealtimeVoiceLogEntry, ctx: RealtimeVoiceLogContext) => void;
|
|
457
509
|
};
|
|
510
|
+
/**
|
|
511
|
+
* The part of a NeuroLink instance a voice agent needs to persist and seed a
|
|
512
|
+
* call's turns. The real `NeuroLink` instance satisfies it.
|
|
513
|
+
*/
|
|
514
|
+
export type LiveKitMemoryNeuroLink = {
|
|
515
|
+
recordConversationTurn: (options: RecordConversationTurnOptions) => Promise<RecordConversationTurnResult>;
|
|
516
|
+
getConversationSeed?: (options: ConversationSeedOptions) => Promise<ConversationSeedMessage[]>;
|
|
517
|
+
};
|
|
518
|
+
/** Memory options for `defineRealtimeVoiceAgent`. */
|
|
519
|
+
export type RealtimeVoiceMemoryConfig = {
|
|
520
|
+
/**
|
|
521
|
+
* Builds the NeuroLink instance, with conversation memory configured the
|
|
522
|
+
* same way the web app's is (same store, key prefix and lifetime), inside
|
|
523
|
+
* the job process. Called once per call, only after a binding verified.
|
|
524
|
+
*/
|
|
525
|
+
createNeuroLink: () => LiveKitMemoryNeuroLink | Promise<LiveKitMemoryNeuroLink>;
|
|
526
|
+
/** Shared secret the binding is signed with. Defaults to `NEUROLINK_VOICE_MEMORY_SECRET`. */
|
|
527
|
+
secret?: string;
|
|
528
|
+
/** How long a closed reply waits for its user transcript before being dropped (default 1500 ms). */
|
|
529
|
+
settleMs?: number;
|
|
530
|
+
/** How long shutdown waits for pending writes (default 800 ms; the worker is reaped about 1 s after shutdown starts). */
|
|
531
|
+
flushBudgetMs?: number;
|
|
532
|
+
/** History seeding at call start. `false` turns it off. Defaults: 10 exchanges, 6000 characters. */
|
|
533
|
+
seed?: false | {
|
|
534
|
+
maxTurns?: number;
|
|
535
|
+
maxChars?: number;
|
|
536
|
+
};
|
|
537
|
+
};
|
|
538
|
+
/** One finished exchange of a voice call, as handed to the writer. */
|
|
539
|
+
export type VoiceTurnRecord = {
|
|
540
|
+
/** The assistant message id, or a synthetic id for a turn cut off by shutdown or barge-in. */
|
|
541
|
+
turnId: string;
|
|
542
|
+
userMessage: string;
|
|
543
|
+
aiResponse: string;
|
|
544
|
+
/** The reply was cut off. */
|
|
545
|
+
interrupted: boolean;
|
|
546
|
+
/** The call ended while this exchange was open. */
|
|
547
|
+
callEnded: boolean;
|
|
548
|
+
startTime: Date;
|
|
549
|
+
/** Tool events of the exchange, in the shape typed chat stores. */
|
|
550
|
+
events?: StreamEventSequence[];
|
|
551
|
+
};
|
|
552
|
+
/** A recorder's current exchange: what has been heard and said since the last reply closed. */
|
|
553
|
+
export type VoiceTurnOpenExchange = {
|
|
554
|
+
userSegments: string[];
|
|
555
|
+
assistantPartial: string;
|
|
556
|
+
assistantStarted: boolean;
|
|
557
|
+
events: StreamEventSequence[];
|
|
558
|
+
startedAt: number;
|
|
559
|
+
};
|
|
560
|
+
/** A reply that finished before its user transcript arrived, waiting out the settle window. */
|
|
561
|
+
export type VoiceTurnClosedExchange = {
|
|
562
|
+
turnId: string;
|
|
563
|
+
assistantText: string;
|
|
564
|
+
interrupted: boolean;
|
|
565
|
+
events: StreamEventSequence[];
|
|
566
|
+
startedAt: number;
|
|
567
|
+
timer: ReturnType<typeof setTimeout>;
|
|
568
|
+
};
|
|
569
|
+
/** The recorder returned by `createVoiceTurnRecorder`. */
|
|
570
|
+
export type VoiceTurnRecorder = {
|
|
571
|
+
/** A final transcript segment of what the user said. Interim segments are ignored. */
|
|
572
|
+
userTranscript: (segment: {
|
|
573
|
+
text: string;
|
|
574
|
+
final: boolean;
|
|
575
|
+
}) => void;
|
|
576
|
+
/** A piece of the assistant's reply as it is produced. */
|
|
577
|
+
assistantDelta: (delta: string) => void;
|
|
578
|
+
/** The assistant's message was added: the reply is complete (or was cut off). */
|
|
579
|
+
assistantMessage: (message: {
|
|
580
|
+
id: string;
|
|
581
|
+
text: string;
|
|
582
|
+
interrupted?: boolean;
|
|
583
|
+
}) => void;
|
|
584
|
+
/**
|
|
585
|
+
* The next assistant-only reply is the agent's opening greeting, not an
|
|
586
|
+
* answer: it is skipped and never paired with the user's first question.
|
|
587
|
+
* Call it once, when the agent is about to speak its greeting.
|
|
588
|
+
*/
|
|
589
|
+
expectGreeting: () => void;
|
|
590
|
+
/** A tool call started in the current exchange. */
|
|
591
|
+
toolStart: (call: {
|
|
592
|
+
name: string;
|
|
593
|
+
args?: unknown;
|
|
594
|
+
}) => void;
|
|
595
|
+
/** A tool call finished in the current exchange. */
|
|
596
|
+
toolEnd: (call: {
|
|
597
|
+
name: string;
|
|
598
|
+
result?: unknown;
|
|
599
|
+
error?: string;
|
|
600
|
+
}) => void;
|
|
601
|
+
/**
|
|
602
|
+
* Shutdown: close what is open, then wait for pending writes — but no longer
|
|
603
|
+
* than `budgetMs`. Safe to call more than once.
|
|
604
|
+
*/
|
|
605
|
+
flush: (budgetMs: number) => Promise<VoiceTurnRecorderStats>;
|
|
606
|
+
/** Counters, for logs and tests. */
|
|
607
|
+
stats: () => VoiceTurnRecorderStats;
|
|
608
|
+
};
|
|
609
|
+
/** What a realtime call needs from its memory once its binding verified. */
|
|
610
|
+
export type RealtimeCallMemory = {
|
|
611
|
+
recorder: VoiceTurnRecorder;
|
|
612
|
+
/** Recent history to seed the model with; empty when there is none or seeding is off. */
|
|
613
|
+
seed: ConversationSeedMessage[];
|
|
614
|
+
/** Late-bound publisher, set once the event bridge exists, for the not-saved notice. */
|
|
615
|
+
setPublisher: (publish: (type: LiveKitVoiceEventType, data: Record<string, unknown>) => void) => void;
|
|
616
|
+
};
|
|
617
|
+
/** Counters of a recorder, for logs and tests. */
|
|
618
|
+
export type VoiceTurnRecorderStats = {
|
|
619
|
+
written: number;
|
|
620
|
+
failed: number;
|
|
621
|
+
dropped: number;
|
|
622
|
+
pending: number;
|
|
623
|
+
};
|
|
624
|
+
/** Options for `createVoiceTurnRecorder`. */
|
|
625
|
+
export type VoiceTurnRecorderOptions = {
|
|
626
|
+
/** Stores one exchange. Throwing means "not stored"; the recorder retries. */
|
|
627
|
+
write: (record: VoiceTurnRecord) => Promise<void>;
|
|
628
|
+
/** A write succeeded. */
|
|
629
|
+
onRecorded?: (record: VoiceTurnRecord) => void;
|
|
630
|
+
/** A write failed after every retry. The call continues. */
|
|
631
|
+
onPermanentFailure?: (record: VoiceTurnRecord, error: unknown) => void;
|
|
632
|
+
/** How long a closed reply waits for its user transcript (default 1500 ms). */
|
|
633
|
+
settleMs?: number;
|
|
634
|
+
/** Retries after the first failed attempt (default 2). */
|
|
635
|
+
retries?: number;
|
|
636
|
+
/** Base delay between attempts; grows linearly (default 250 ms). */
|
|
637
|
+
retryDelayMs?: number;
|
|
638
|
+
/** Appended to a reply that was cut off by the call ending. */
|
|
639
|
+
callEndedNote?: string;
|
|
640
|
+
/** Stored as the reply when the call ended with the user's turn unanswered. */
|
|
641
|
+
noReplyNote?: string;
|
|
642
|
+
/** Prefix of synthetic turn ids (default: a random id). */
|
|
643
|
+
callId?: string;
|
|
644
|
+
/** Clock, for tests. */
|
|
645
|
+
now?: () => number;
|
|
646
|
+
};
|
|
647
|
+
/** A signed memory binding as it appears in room metadata under `memory`. */
|
|
648
|
+
export type VoiceMemoryBinding = {
|
|
649
|
+
userId: string;
|
|
650
|
+
sessionId: string;
|
|
651
|
+
/** Expiry, Unix seconds. */
|
|
652
|
+
exp: number;
|
|
653
|
+
/** base64url(HMAC-SHA256) over the room name, userId, sessionId and exp. */
|
|
654
|
+
sig: string;
|
|
655
|
+
};
|
|
656
|
+
/** Input to `signVoiceMemoryBinding`. */
|
|
657
|
+
export type VoiceMemoryBindingInput = {
|
|
658
|
+
/** The LiveKit room name the binding is for. */
|
|
659
|
+
room: string;
|
|
660
|
+
userId: string;
|
|
661
|
+
sessionId: string;
|
|
662
|
+
secret: string;
|
|
663
|
+
/** Lifetime in seconds (default 7200, at most 86400). */
|
|
664
|
+
ttlSeconds?: number;
|
|
665
|
+
/** Current time, Unix seconds (default: now). */
|
|
666
|
+
now?: number;
|
|
667
|
+
};
|
|
668
|
+
/** Input to `verifyVoiceMemoryBinding`. */
|
|
669
|
+
export type VoiceMemoryBindingVerifyInput = {
|
|
670
|
+
/** The room's base64(JSON) metadata string. */
|
|
671
|
+
roomMetadata: string | undefined;
|
|
672
|
+
/** The room name as LiveKit reports it for the joined room. */
|
|
673
|
+
roomName: string | undefined;
|
|
674
|
+
secret: string | undefined;
|
|
675
|
+
/** Current time, Unix seconds (default: now). */
|
|
676
|
+
now?: number;
|
|
677
|
+
};
|
|
678
|
+
/** Why a binding was not accepted. */
|
|
679
|
+
export type VoiceMemoryBindingRejection = "no-secret" | "no-binding" | "malformed" | "bad-signature" | "expired" | "expiry-too-far";
|
|
680
|
+
/** Result of `verifyVoiceMemoryBinding`. */
|
|
681
|
+
export type VoiceMemoryBindingVerdict = {
|
|
682
|
+
ok: true;
|
|
683
|
+
userId: string;
|
|
684
|
+
sessionId: string;
|
|
685
|
+
exp: number;
|
|
686
|
+
} | {
|
|
687
|
+
ok: false;
|
|
688
|
+
reason: VoiceMemoryBindingRejection;
|
|
689
|
+
};
|
|
458
690
|
/** Auth token + base64 MCP execution context decoded from a room's metadata. */
|
|
459
691
|
export type LiveKitRoomCallContext = {
|
|
460
692
|
/** Lighthouse access JWT used as `x-auth-token` to the MCP server. */
|
|
@@ -509,4 +741,15 @@ export type BuildRealtimeMcpToolsParams = {
|
|
|
509
741
|
* model can talk about.
|
|
510
742
|
*/
|
|
511
743
|
toolTimeoutMs?: number;
|
|
744
|
+
/** Observes a tool call as it starts. A throwing observer never affects the call. */
|
|
745
|
+
onToolStart?: (call: {
|
|
746
|
+
name: string;
|
|
747
|
+
args: Record<string, unknown>;
|
|
748
|
+
}) => void;
|
|
749
|
+
/** Observes a tool call as it ends, with its result as text or its error. */
|
|
750
|
+
onToolEnd?: (call: {
|
|
751
|
+
name: string;
|
|
752
|
+
result?: string;
|
|
753
|
+
error?: string;
|
|
754
|
+
}) => void;
|
|
512
755
|
};
|
package/dist/types/sdk.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export type { AnalyticsData, TokenUsage } from "./analytics.js";
|
|
|
12
12
|
export type { BaseCommandArgs, CommandResult, GenerateCommandArgs, CliGenerateResult, MCPCommandArgs, ModelsCommandArgs, CliStreamChunk, } from "./cli.js";
|
|
13
13
|
export type { ErrorInfo, FunctionParameters, JsonArray, JsonObject, JsonValue, ProcessResult, Result, TestFunction, TestResult, Unknown, UnknownArray, UnknownRecord, } from "./common.js";
|
|
14
14
|
export type { Content, ImageContent, MultimodalMessage, ProcessedImage, ProviderImageFormat, ProviderMultimodalPayload, TextContent, VisionCapability, } from "./content.js";
|
|
15
|
-
export type { ChatMessage, ChatMessageMetadata, ToolResultData, ConversationMemoryConfig, ConversationMemoryError, ConversationMemoryEvents, MessageContent, MultimodalChatMessage, RedisConversationObject, RedisStorageConfig, SessionIdentifier, SessionMemory, SessionMetadata, StoreConversationTurnOptions, } from "./conversation.js";
|
|
15
|
+
export type { ChatMessage, ChatMessageMetadata, ToolResultData, ConversationMemoryConfig, ConversationMemoryError, ConversationMemoryEvents, ConversationModality, ConversationSeedMessage, ConversationSeedOptions, MessageContent, MultimodalChatMessage, RecordConversationTurnOptions, RecordConversationTurnResult, RedisConversationObject, RedisStorageConfig, SessionIdentifier, SessionMemory, SessionMetadata, StoreConversationTurnOptions, } from "./conversation.js";
|
|
16
16
|
export type { DomainConfig, DomainConfigOptions, DomainEvaluationCriteria, DomainTemplate, DomainType, DomainValidationRule, } from "./domain.js";
|
|
17
17
|
export type { EnhancedEvaluationResult, EvaluationContext, EvaluationCriteria, EvaluationData, EvaluationRequest, } from "./evaluation.js";
|
|
18
18
|
export type { ExternalMCPConfigValidation, ExternalMCPManagerConfig, ExternalMCPOperationResult, ExternalMCPServerEvents, ExternalMCPServerHealth, ExternalMCPServerInstance, ExternalMCPServerStatus, ExternalMCPToolContext, ExternalMCPToolInfo, ExternalMCPToolResult, } from "./externalMcp.js";
|
package/dist/types/stream.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ import type { TokenUsage } from "./analytics.js";
|
|
|
9
9
|
import type { JsonValue, UnknownRecord } from "./common.js";
|
|
10
10
|
import type { TerminalAgentModeOption, TerminalAgentModeVersion } from "./agentMode.js";
|
|
11
11
|
import type { Content, ImageWithAltText } from "./content.js";
|
|
12
|
-
import type { ChatMessage, ToolReplayMode } from "./conversation.js";
|
|
12
|
+
import type { ChatMessage, ConversationModality, ToolReplayMode } from "./conversation.js";
|
|
13
13
|
import type { StreamNoOutputSentinel } from "./noOutputSentinel.js";
|
|
14
14
|
import type { GenerateStopReason, TTSMetadata, ToolExecutionCaptureOptions } from "./generate.js";
|
|
15
15
|
import type { MemoryCallOptions } from "./memory.js";
|
|
@@ -666,6 +666,13 @@ export type StreamOptions = {
|
|
|
666
666
|
workflow?: string;
|
|
667
667
|
workflowConfig?: WorkflowConfig;
|
|
668
668
|
enableSummarization?: boolean;
|
|
669
|
+
/**
|
|
670
|
+
* How this turn was entered. Stamped on the stored messages' metadata so
|
|
671
|
+
* history readers can tell a spoken turn from a typed one. Leave it unset
|
|
672
|
+
* for typed input: the stored turn is then identical to what it was before
|
|
673
|
+
* the field existed.
|
|
674
|
+
*/
|
|
675
|
+
modality?: ConversationModality;
|
|
669
676
|
/**
|
|
670
677
|
* Maximum cumulative cost (USD) for this session.
|
|
671
678
|
* Once the session spend reaches this limit, subsequent stream() calls
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Message identity and metadata for a stored conversation turn.
|
|
3
|
+
*
|
|
4
|
+
* Both memory managers store a turn as a user message followed by an assistant
|
|
5
|
+
* message. When the caller supplies a `turnId` the two messages get stable ids
|
|
6
|
+
* (`${turnId}:user`, `${turnId}:assistant`), which lets a repeated store of the
|
|
7
|
+
* same turn be recognised and skipped. When the caller sets neither a
|
|
8
|
+
* `turnId` nor a `modality`, nothing here changes what is stored: ids stay
|
|
9
|
+
* random and no metadata is added, so typed chat is byte-for-byte as before.
|
|
10
|
+
*/
|
|
11
|
+
import type { ChatMessage, ChatMessageMetadata, StoreConversationTurnOptions, TurnMessageIdentity } from "../types/index.js";
|
|
12
|
+
export declare function userTurnMessageId(turnId: string): string;
|
|
13
|
+
export declare function assistantTurnMessageId(turnId: string): string;
|
|
14
|
+
/** Identity and metadata for the user message of a turn. */
|
|
15
|
+
export declare function userTurnIdentity(options: Pick<StoreConversationTurnOptions, "turnId" | "modality">): TurnMessageIdentity;
|
|
16
|
+
/** Identity and metadata for the assistant message of a turn. */
|
|
17
|
+
export declare function assistantTurnIdentity(options: Pick<StoreConversationTurnOptions, "turnId" | "modality" | "interrupted" | "callEnded">, thoughtSignature?: string): TurnMessageIdentity;
|
|
18
|
+
/** `id` and optional `metadata` of a stored message, ready to spread into it. */
|
|
19
|
+
export declare function turnMessageFields(identity: TurnMessageIdentity): {
|
|
20
|
+
id: string;
|
|
21
|
+
metadata?: ChatMessageMetadata;
|
|
22
|
+
};
|
|
23
|
+
/** True when a turn with this id is already among the session's recent messages. */
|
|
24
|
+
export declare function isTurnAlreadyStored(messages: ReadonlyArray<Pick<ChatMessage, "id">>, turnId: string | undefined): boolean;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Message identity and metadata for a stored conversation turn.
|
|
3
|
+
*
|
|
4
|
+
* Both memory managers store a turn as a user message followed by an assistant
|
|
5
|
+
* message. When the caller supplies a `turnId` the two messages get stable ids
|
|
6
|
+
* (`${turnId}:user`, `${turnId}:assistant`), which lets a repeated store of the
|
|
7
|
+
* same turn be recognised and skipped. When the caller sets neither a
|
|
8
|
+
* `turnId` nor a `modality`, nothing here changes what is stored: ids stay
|
|
9
|
+
* random and no metadata is added, so typed chat is byte-for-byte as before.
|
|
10
|
+
*/
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
12
|
+
import { logger } from "./logger.js";
|
|
13
|
+
/** How far back a repeated `turnId` is looked for. A turn is always the newest thing in a session. */
|
|
14
|
+
const DUPLICATE_LOOKBACK_MESSAGES = 200;
|
|
15
|
+
export function userTurnMessageId(turnId) {
|
|
16
|
+
return `${turnId}:user`;
|
|
17
|
+
}
|
|
18
|
+
export function assistantTurnMessageId(turnId) {
|
|
19
|
+
return `${turnId}:assistant`;
|
|
20
|
+
}
|
|
21
|
+
/** Identity and metadata for the user message of a turn. */
|
|
22
|
+
export function userTurnIdentity(options) {
|
|
23
|
+
const metadata = {};
|
|
24
|
+
if (options.modality) {
|
|
25
|
+
metadata.modality = options.modality;
|
|
26
|
+
}
|
|
27
|
+
if (options.turnId) {
|
|
28
|
+
metadata.turnId = options.turnId;
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
id: options.turnId ? userTurnMessageId(options.turnId) : undefined,
|
|
32
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/** Identity and metadata for the assistant message of a turn. */
|
|
36
|
+
export function assistantTurnIdentity(options, thoughtSignature) {
|
|
37
|
+
const metadata = {};
|
|
38
|
+
if (thoughtSignature) {
|
|
39
|
+
metadata.thoughtSignature = thoughtSignature;
|
|
40
|
+
}
|
|
41
|
+
if (options.modality) {
|
|
42
|
+
metadata.modality = options.modality;
|
|
43
|
+
}
|
|
44
|
+
if (options.turnId) {
|
|
45
|
+
metadata.turnId = options.turnId;
|
|
46
|
+
}
|
|
47
|
+
if (options.interrupted) {
|
|
48
|
+
metadata.interrupted = true;
|
|
49
|
+
}
|
|
50
|
+
if (options.callEnded) {
|
|
51
|
+
metadata.callEnded = true;
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
id: options.turnId ? assistantTurnMessageId(options.turnId) : undefined,
|
|
55
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** `id` and optional `metadata` of a stored message, ready to spread into it. */
|
|
59
|
+
export function turnMessageFields(identity) {
|
|
60
|
+
return {
|
|
61
|
+
id: identity.id ?? randomUUID(),
|
|
62
|
+
...(identity.metadata ? { metadata: identity.metadata } : {}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** True when a turn with this id is already among the session's recent messages. */
|
|
66
|
+
export function isTurnAlreadyStored(messages, turnId) {
|
|
67
|
+
if (!turnId) {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
const assistantId = assistantTurnMessageId(turnId);
|
|
71
|
+
const start = Math.max(0, messages.length - DUPLICATE_LOOKBACK_MESSAGES);
|
|
72
|
+
for (let index = messages.length - 1; index >= start; index -= 1) {
|
|
73
|
+
if (messages[index].id === assistantId) {
|
|
74
|
+
logger.debug("[conversationTurnIdentity] Turn already stored, skipping", {
|
|
75
|
+
turnId,
|
|
76
|
+
});
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
@@ -37,16 +37,17 @@ function extractTextDelta(chunk) {
|
|
|
37
37
|
export function createVoiceBrain(config) {
|
|
38
38
|
const { neurolink, provider, model, systemPrompt, temperature, maxTokens, userId, } = config;
|
|
39
39
|
async function* streamReply(turn) {
|
|
40
|
-
const { transcript, conversationId, signal } = turn;
|
|
40
|
+
const { transcript, conversationId, signal, ephemeral } = turn;
|
|
41
41
|
if (signal?.aborted) {
|
|
42
42
|
return;
|
|
43
43
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
44
|
+
// No sessionId means no history read and no turn stored for this call.
|
|
45
|
+
const context = ephemeral
|
|
46
|
+
? {}
|
|
47
|
+
: { sessionId: conversationId, conversationId };
|
|
48
|
+
const turnUserId = turn.userId ?? userId;
|
|
49
|
+
if (!ephemeral && turnUserId !== undefined) {
|
|
50
|
+
context.userId = turnUserId;
|
|
50
51
|
}
|
|
51
52
|
const result = await neurolink.stream({
|
|
52
53
|
input: { text: transcript },
|
|
@@ -56,6 +57,8 @@ export function createVoiceBrain(config) {
|
|
|
56
57
|
temperature,
|
|
57
58
|
maxTokens,
|
|
58
59
|
context,
|
|
60
|
+
// Every turn through this brain is spoken; stored messages are marked so.
|
|
61
|
+
modality: "voice",
|
|
59
62
|
abortSignal: signal,
|
|
60
63
|
disableTools: false,
|
|
61
64
|
});
|
|
@@ -18,5 +18,7 @@ export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
|
|
|
18
18
|
export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
|
|
19
19
|
export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
|
|
20
20
|
export { readCallContextFromRoom } from "./roomContext.js";
|
|
21
|
+
export { signVoiceMemoryBinding, verifyVoiceMemoryBinding, isValidVoiceSessionId, isValidVoiceUserId, VOICE_MEMORY_SECRET_ENV, VOICE_MEMORY_BINDING_VERSION, VOICE_MEMORY_MAX_TTL_SECONDS, } from "./memoryBinding.js";
|
|
22
|
+
export { createVoiceTurnRecorder } from "./turnRecorder.js";
|
|
21
23
|
export { sanitizeSchema, sanitizeToolParameters, findSchemaIssue, } from "./schemaSanitizer.js";
|
|
22
24
|
export { buildRealtimeMcpTools, mcpResultToText } from "./realtimeMcpTools.js";
|
|
@@ -18,5 +18,7 @@ export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
|
|
|
18
18
|
export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
|
|
19
19
|
export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
|
|
20
20
|
export { readCallContextFromRoom } from "./roomContext.js";
|
|
21
|
+
export { signVoiceMemoryBinding, verifyVoiceMemoryBinding, isValidVoiceSessionId, isValidVoiceUserId, VOICE_MEMORY_SECRET_ENV, VOICE_MEMORY_BINDING_VERSION, VOICE_MEMORY_MAX_TTL_SECONDS, } from "./memoryBinding.js";
|
|
22
|
+
export { createVoiceTurnRecorder } from "./turnRecorder.js";
|
|
21
23
|
export { sanitizeSchema, sanitizeToolParameters, findSchemaIssue, } from "./schemaSanitizer.js";
|
|
22
24
|
export { buildRealtimeMcpTools, mcpResultToText } from "./realtimeMcpTools.js";
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Signed memory binding for a voice call.
|
|
3
|
+
*
|
|
4
|
+
* A voice worker persists a call's turns to a conversation, so it has to know
|
|
5
|
+
* WHOSE conversation and WHICH one — and it must not take either from anything
|
|
6
|
+
* a participant can influence. The manager that creates the room (for example
|
|
7
|
+
* a `/start` endpoint) already holds the verified session, so it signs a small
|
|
8
|
+
* binding and writes it into the room metadata next to `authToken` and
|
|
9
|
+
* `mcpContext`:
|
|
10
|
+
*
|
|
11
|
+
* base64(JSON({ authToken, mcpContext, memory: { userId, sessionId, exp, sig } }))
|
|
12
|
+
*
|
|
13
|
+
* The worker verifies the signature with the same shared secret. The signature
|
|
14
|
+
* covers the room name, so a binding copied into another room does not verify.
|
|
15
|
+
* Room metadata is readable by participants, so this does not hide the ids; it
|
|
16
|
+
* stops anyone without the secret from choosing them.
|
|
17
|
+
*
|
|
18
|
+
* Everything here fails closed. A missing secret, a missing or malformed
|
|
19
|
+
* binding, a bad signature or an expired binding all yield `{ ok: false }`, and
|
|
20
|
+
* the caller carries on with the call, storing nothing.
|
|
21
|
+
*
|
|
22
|
+
* See docs/features/livekit-voice-agent.md.
|
|
23
|
+
*/
|
|
24
|
+
import type { VoiceMemoryBinding, VoiceMemoryBindingInput, VoiceMemoryBindingVerdict, VoiceMemoryBindingVerifyInput } from "../../types/index.js";
|
|
25
|
+
/** Domain-separation label that starts every signed message. */
|
|
26
|
+
export declare const VOICE_MEMORY_BINDING_VERSION = "neurolink.voice.memory.v1";
|
|
27
|
+
/** Environment variable the worker reads the shared secret from when none is passed in. */
|
|
28
|
+
export declare const VOICE_MEMORY_SECRET_ENV = "NEUROLINK_VOICE_MEMORY_SECRET";
|
|
29
|
+
/** Longest expiry, counted from now, that a worker accepts (24 hours). */
|
|
30
|
+
export declare const VOICE_MEMORY_MAX_TTL_SECONDS: number;
|
|
31
|
+
/** A conversation id a voice call may append to: letters, digits, "_" and "-", up to 128, no ":". */
|
|
32
|
+
export declare function isValidVoiceSessionId(value: unknown): value is string;
|
|
33
|
+
/** A user id a voice call may be bound to: 1 to 256 characters, no whitespace or control characters. */
|
|
34
|
+
export declare function isValidVoiceUserId(value: unknown): value is string;
|
|
35
|
+
/**
|
|
36
|
+
* Sign a binding for a room. Throws on an unusable secret or invalid ids: the
|
|
37
|
+
* signer is the trusted side, and a binding it cannot sign correctly is a bug
|
|
38
|
+
* to surface there, not something to paper over.
|
|
39
|
+
*/
|
|
40
|
+
export declare function signVoiceMemoryBinding(input: VoiceMemoryBindingInput): VoiceMemoryBinding;
|
|
41
|
+
/**
|
|
42
|
+
* Read and verify the memory binding in a room's metadata.
|
|
43
|
+
*
|
|
44
|
+
* Never throws and never logs ids. `reason` says why a binding was rejected so
|
|
45
|
+
* the caller can log it.
|
|
46
|
+
*/
|
|
47
|
+
export declare function verifyVoiceMemoryBinding(input: VoiceMemoryBindingVerifyInput): VoiceMemoryBindingVerdict;
|