@juspay/neurolink 12.49.20 → 12.51.0

package/dist/voice/livekit/realtimeVoiceAgent.js

@@ -19,6 +19,7 @@ import { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
 import { readCallContextFromRoom } from "./roomContext.js";
 import { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
 import { buildRealtimeMcpTools } from "./realtimeMcpTools.js";
+import { setUpRealtimeCallMemory } from "./realtimeMemory.js";
 const realtimeLogEventSchema = z.object({
     level: z.enum(["debug", "info", "warn", "error"]),
     message: z.string(),
@@ -144,6 +145,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
     const eventsTopic = config.eventsTopic;
     const controlTopic = config.controlTopic;
     const onLog = config.onLog;
+    const memoryConfig = config.memory;
     async function entry(ctx) {
         // The Gemini Live WS authenticates to Vertex via ADC; materialise it and
         // force ADC (not an API key) auth before any realtime connection.
@@ -192,6 +194,9 @@ export function defineRealtimeVoiceAgent(config = {}) {
             location: cfg.location,
             ms: sinceConnect(),
         });
+        // Decide whether this call may be remembered BEFORE the lifecycle installs
+        // its process reaper, so the shutdown flush is registered first.
+        const memory = await setUpRealtimeCallMemory(ctx, memoryConfig);
         await installRealtimeJobLifecycle(ctx, cfg);
         const { voice } = await import("@livekit/agents");
         const google = await import("@livekit/agents-plugin-google");
@@ -205,6 +210,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
         ctx.addShutdownCallback(async () => {
             bridge.dispose();
         });
+        memory?.setPublisher(bridge.publishEvent);
         let agentTools;
         if (cfg.toolsEnabled) {
             const { authToken, xContext } = readCallContextFromRoom(ctx.room.metadata);
@@ -222,6 +228,12 @@ export function defineRealtimeVoiceAgent(config = {}) {
                         xContext,
                         publishEvent: bridge.publishEvent,
                         requestConfirmation: bridge.requestConfirmation,
+                        ...(memory
+                            ? {
+                                onToolStart: (call) => memory.recorder.toolStart(call),
+                                onToolEnd: (call) => memory.recorder.toolEnd(call),
+                            }
+                            : {}),
                     });
                     agentTools = toolset.tools;
                     logger.info("realtime.mcp.enabled", {
@@ -265,9 +277,23 @@ export function defineRealtimeVoiceAgent(config = {}) {
         const session = new voice.AgentSession({
             llm: new google.realtime.RealtimeModel(modelOptions),
         });
+        // The conversation so far, so a call that appends to an open conversation
+        // starts knowing it. Empty (and omitted) when there is nothing to seed.
+        let seededChatCtx;
+        if (memory && memory.seed.length > 0) {
+            const { llm } = await import("@livekit/agents");
+            seededChatCtx = llm.ChatContext.empty();
+            for (const message of memory.seed) {
+                seededChatCtx.addMessage({
+                    role: message.role,
+                    content: message.content,
+                });
+            }
+        }
         const agent = new voice.Agent({
             instructions: cfg.systemPrompt,
             ...(agentTools ? { tools: agentTools } : {}),
+            ...(seededChatCtx ? { chatCtx: seededChatCtx } : {}),
         });
         agent.transcriptionNode = async (textStream, _modelSettings) => {
             const [forUi, forDownstream] = textStream.tee();
@@ -290,6 +316,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
                                 });
                             }
                             bridge.publishEvent("text", { delta: chunk });
+                            memory?.recorder.assistantDelta(chunk);
                         }
                     }
                 }
@@ -328,6 +355,10 @@ export function defineRealtimeVoiceAgent(config = {}) {
                 text: ev.transcript,
                 final: ev.isFinal,
             });
+            memory?.recorder.userTranscript({
+                text: ev.transcript,
+                final: ev.isFinal,
+            });
         });
         session.on(voice.AgentSessionEventTypes.AgentStateChanged, (ev) => {
             logger.debug("realtime.session.stateChanged", {
@@ -342,7 +373,14 @@ export function defineRealtimeVoiceAgent(config = {}) {
                 ev.item.role === "assistant" &&
                 ev.item.textContent) {
                 logger.debug("realtime.session.turnDone", { ms: sinceConnect() });
-                bridge.publishEvent("done", {});
+                // The id the stored messages carry, so a client can merge its live
+                // copy of the turn with history instead of showing it twice.
+                bridge.publishEvent("done", memory ? { turnId: ev.item.id } : {});
+                memory?.recorder.assistantMessage({
+                    id: ev.item.id,
+                    text: ev.item.textContent,
+                    interrupted: ev.item.interrupted,
+                });
             }
         });
         session.on(voice.AgentSessionEventTypes.Error, (ev) => {
@@ -357,6 +395,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
         bridge.publishEvent("status", { state: "listening" });
         if (cfg.greeting.trim().length > 0) {
             logger.info("realtime.session.greeting", { ms: sinceConnect() });
+            memory?.recorder.expectGreeting();
             session.generateReply({ instructions: cfg.greeting });
         }
     }

package/dist/voice/livekit/turnRecorder.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Turns a live voice call's event stream into stored conversation turns.
+ *
+ * A realtime voice model reports a call as separate, loosely ordered events:
+ * final user transcripts, assistant text deltas, tool calls, and the added
+ * assistant message that closes a reply. This recorder groups them into
+ * exchanges and hands each finished exchange to a `write` function (normally
+ * `NeuroLink.recordConversationTurn`).
+ *
+ * It knows nothing about LiveKit, so it can be driven by a synthetic event
+ * stream in tests. The rules it applies:
+ *
+ *  - An exchange is the user's final transcript(s) plus the assistant message
+ *    that answers them. The assistant message's id is the exchange's `turnId`,
+ *    which makes a retried write idempotent.
+ *  - Settle rule. A user's final transcript can arrive after the assistant has
+ *    started, or even finished, replying. A closed reply with no user text is
+ *    held for `settleMs`; a transcript arriving in that window completes it.
+ *    A reply still without user text after the window (the opening greeting,
+ *    typically) is dropped, because a stored turn needs both sides.
+ *  - A new user transcript arriving while the previous reply is still being
+ *    spoken, and after that exchange already had user text, closes the
+ *    previous exchange as interrupted (barge-in) and starts a new one.
+ *  - `flush` is for shutdown. It closes the open exchange with what was said
+ *    so far, marks it interrupted and call-ended, and appends a short note so
+ *    a later reader knows the reply is incomplete. A user turn with no reply
+ *    gets the note alone. The flush waits for pending writes only up to a
+ *    budget, because the worker process is reaped about a second after the
+ *    shutdown callbacks start.
+ *  - Writes run strictly one at a time, in order. A failed write is retried;
+ *    after the last retry it is reported through `onPermanentFailure` and the
+ *    recorder carries on — a storage problem never stops a call.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import type { VoiceTurnRecorder, VoiceTurnRecorderOptions } from "../../types/index.js";
+/** Create a recorder for one call. */
+export declare function createVoiceTurnRecorder(options: VoiceTurnRecorderOptions): VoiceTurnRecorder;

package/dist/voice/livekit/turnRecorder.js

@@ -0,0 +1,362 @@
+/**
+ * Turns a live voice call's event stream into stored conversation turns.
+ *
+ * A realtime voice model reports a call as separate, loosely ordered events:
+ * final user transcripts, assistant text deltas, tool calls, and the added
+ * assistant message that closes a reply. This recorder groups them into
+ * exchanges and hands each finished exchange to a `write` function (normally
+ * `NeuroLink.recordConversationTurn`).
+ *
+ * It knows nothing about LiveKit, so it can be driven by a synthetic event
+ * stream in tests. The rules it applies:
+ *
+ *  - An exchange is the user's final transcript(s) plus the assistant message
+ *    that answers them. The assistant message's id is the exchange's `turnId`,
+ *    which makes a retried write idempotent.
+ *  - Settle rule. A user's final transcript can arrive after the assistant has
+ *    started, or even finished, replying. A closed reply with no user text is
+ *    held for `settleMs`; a transcript arriving in that window completes it.
+ *    A reply still without user text after the window (the opening greeting,
+ *    typically) is dropped, because a stored turn needs both sides.
+ *  - A new user transcript arriving while the previous reply is still being
+ *    spoken, and after that exchange already had user text, closes the
+ *    previous exchange as interrupted (barge-in) and starts a new one.
+ *  - `flush` is for shutdown. It closes the open exchange with what was said
+ *    so far, marks it interrupted and call-ended, and appends a short note so
+ *    a later reader knows the reply is incomplete. A user turn with no reply
+ *    gets the note alone. The flush waits for pending writes only up to a
+ *    budget, because the worker process is reaped about a second after the
+ *    shutdown callbacks start.
+ *  - Writes run strictly one at a time, in order. A failed write is retried;
+ *    after the last retry it is reported through `onPermanentFailure` and the
+ *    recorder carries on — a storage problem never stops a call.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import { randomUUID } from "node:crypto";
+import { logger } from "../../utils/logger.js";
+const DEFAULT_SETTLE_MS = 1500;
+const DEFAULT_RETRIES = 2;
+const DEFAULT_RETRY_DELAY_MS = 250;
+const DEFAULT_CALL_ENDED_NOTE = "[The call ended before this reply finished.]";
+const DEFAULT_NO_REPLY_NOTE = "[The call ended before a reply.]";
+/** Tool arguments and results are stored with the turn; keep one entry's text bounded. */
+const MAX_TOOL_TEXT_CHARS = 4000;
+function boundedText(value) {
+    return value.length > MAX_TOOL_TEXT_CHARS
+        ? `${value.slice(0, MAX_TOOL_TEXT_CHARS)}…[truncated]`
+        : value;
+}
+function boundedPayload(value) {
+    if (typeof value === "string") {
+        return boundedText(value);
+    }
+    try {
+        const text = JSON.stringify(value);
+        return text !== undefined && text.length > MAX_TOOL_TEXT_CHARS
+            ? boundedText(text)
+            : value;
+    }
+    catch {
+        return "[unserializable]";
+    }
+}
+function wait(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+/**
+ * Store one exchange, retrying with a growing delay. After the last attempt (or
+ * once shutdown has run out of time) the failure is counted, logged and handed
+ * to `onPermanentFailure`; nothing here throws, so a storage problem never
+ * stops a call.
+ */
+async function writeWithRetry(record, context) {
+    const { options, stats, retries, retryDelayMs, isOutOfTime } = context;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt += 1) {
+        try {
+            await options.write(record);
+            stats.written += 1;
+            options.onRecorded?.(record);
+            return;
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt === retries || isOutOfTime()) {
+                break;
+            }
+            await wait(retryDelayMs * (attempt + 1));
+        }
+    }
+    stats.failed += 1;
+    logger.error("voice.turnRecorder.writeFailed", {
+        turnId: record.turnId,
+        error: lastError instanceof Error ? lastError.message : String(lastError),
+    });
+    try {
+        options.onPermanentFailure?.(record, lastError);
+    }
+    catch {
+        /* a failure sink must never break the call */
+    }
+}
+/** Create a recorder for one call. */
+export function createVoiceTurnRecorder(options) {
+    const settleMs = options.settleMs ?? DEFAULT_SETTLE_MS;
+    const retries = options.retries ?? DEFAULT_RETRIES;
+    const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+    const callEndedNote = options.callEndedNote ?? DEFAULT_CALL_ENDED_NOTE;
+    const noReplyNote = options.noReplyNote ?? DEFAULT_NO_REPLY_NOTE;
+    const now = options.now ?? Date.now;
+    const callId = options.callId ?? randomUUID();
+    const stats = {
+        written: 0,
+        failed: 0,
+        dropped: 0,
+        pending: 0,
+    };
+    let open = newExchange();
+    // Every reply id that has been written or queued, so a reply delivered a
+    // second time can never be paired with the next question.
+    const seenReplyIds = new Set();
+    let greetingPending = false;
+    const awaitingUser = [];
+    let queue = Promise.resolve();
+    let eventSeq = 0;
+    let syntheticTurns = 0;
+    let closing = false;
+    let deadline;
+    function newExchange() {
+        return {
+            userSegments: [],
+            assistantPartial: "",
+            assistantStarted: false,
+            events: [],
+            startedAt: now(),
+        };
+    }
+    function nextEvent(type, fields) {
+        const event = {
+            type,
+            seq: eventSeq,
+            timestamp: now(),
+            ...fields,
+        };
+        eventSeq += 1;
+        return event;
+    }
+    function enqueue(record) {
+        stats.pending += 1;
+        queue = queue.then(async () => {
+            try {
+                await writeWithRetry(record, {
+                    options,
+                    stats,
+                    retries,
+                    retryDelayMs,
+                    isOutOfTime: () => closing && deadline !== undefined && now() >= deadline,
+                });
+            }
+            finally {
+                stats.pending -= 1;
+            }
+        });
+    }
+    function finishExchange(exchange) {
+        enqueue({
+            turnId: exchange.turnId,
+            userMessage: exchange.user,
+            aiResponse: exchange.assistant,
+            interrupted: exchange.interrupted,
+            callEnded: exchange.callEnded,
+            startTime: new Date(exchange.startedAt),
+            ...(exchange.events.length > 0 ? { events: exchange.events } : {}),
+        });
+    }
+    function syntheticTurnId() {
+        syntheticTurns += 1;
+        return `call-${callId}-${syntheticTurns}`;
+    }
+    function dropClosed(exchange) {
+        clearTimeout(exchange.timer);
+        stats.dropped += 1;
+        logger.debug("voice.turnRecorder.assistantOnlyDropped", {
+            turnId: exchange.turnId,
+        });
+    }
+    return {
+        userTranscript(segment) {
+            const text = segment.text.trim();
+            if (closing || !segment.final || !text) {
+                return;
+            }
+            const waiting = awaitingUser.shift();
+            if (waiting) {
+                clearTimeout(waiting.timer);
+                finishExchange({
+                    turnId: waiting.turnId,
+                    user: text,
+                    assistant: waiting.assistantText,
+                    interrupted: waiting.interrupted,
+                    callEnded: false,
+                    events: waiting.events,
+                    startedAt: waiting.startedAt,
+                });
+                return;
+            }
+            if (open.assistantStarted && open.userSegments.length > 0) {
+                // The user spoke again while the previous reply was still going: close
+                // that exchange as cut off, and start a new one with this transcript.
+                finishExchange({
+                    turnId: syntheticTurnId(),
+                    user: open.userSegments.join(" "),
+                    assistant: open.assistantPartial.trim() || noReplyNote,
+                    interrupted: true,
+                    callEnded: false,
+                    events: open.events,
+                    startedAt: open.startedAt,
+                });
+                open = newExchange();
+            }
+            open.userSegments.push(text);
+        },
+        assistantDelta(delta) {
+            if (closing || !delta) {
+                return;
+            }
+            open.assistantStarted = true;
+            open.assistantPartial += delta;
+        },
+        assistantMessage(message) {
+            if (closing) {
+                return;
+            }
+            const text = message.text.trim() || open.assistantPartial.trim();
+            if (!text) {
+                // An item with no spoken text (a tool-only message) closes nothing.
+                return;
+            }
+            if (seenReplyIds.has(message.id)) {
+                // The same reply delivered again, whether it is still waiting for its
+                // user text or was already written.
+                return;
+            }
+            seenReplyIds.add(message.id);
+            if (greetingPending) {
+                // The agent's opening line is an instruction, not an answer: it is not
+                // a turn, and it must never be paired with the user's first question
+                // (they may speak over it).
+                greetingPending = false;
+                // Whatever the user has already said stays: it belongs to the reply
+                // that follows, not to the greeting.
+                open.assistantPartial = "";
+                open.assistantStarted = false;
+                return;
+            }
+            const exchange = open;
+            open = newExchange();
+            const interrupted = message.interrupted === true;
+            if (exchange.userSegments.length > 0) {
+                finishExchange({
+                    turnId: message.id,
+                    user: exchange.userSegments.join(" "),
+                    assistant: text,
+                    interrupted,
+                    callEnded: false,
+                    events: exchange.events,
+                    startedAt: exchange.startedAt,
+                });
+                return;
+            }
+            const closed = {
+                turnId: message.id,
+                assistantText: text,
+                interrupted,
+                events: exchange.events,
+                startedAt: exchange.startedAt,
+                timer: setTimeout(() => {
+                    const index = awaitingUser.indexOf(closed);
+                    if (index >= 0) {
+                        awaitingUser.splice(index, 1);
+                        dropClosed(closed);
+                    }
+                }, settleMs),
+            };
+            closed.timer.unref?.();
+            awaitingUser.push(closed);
+        },
+        expectGreeting() {
+            greetingPending = true;
+        },
+        toolStart(call) {
+            if (closing) {
+                return;
+            }
+            open.events.push(nextEvent("tool:start", {
+                tool: call.name,
+                toolName: call.name,
+                ...(call.args !== undefined
+                    ? { args: boundedPayload(call.args) }
+                    : {}),
+            }));
+        },
+        toolEnd(call) {
+            if (closing) {
+                return;
+            }
+            open.events.push(nextEvent("tool:end", {
+                tool: call.name,
+                toolName: call.name,
+                success: call.error === undefined,
+                ...(call.error !== undefined
+                    ? { error: boundedText(call.error) }
+                    : { result: boundedPayload(call.result) }),
+            }));
+        },
+        async flush(budgetMs) {
+            if (!closing) {
+                closing = true;
+                deadline = now() + budgetMs;
+                for (const waiting of awaitingUser.splice(0)) {
+                    dropClosed(waiting);
+                }
+                const user = open.userSegments.join(" ");
+                if (user) {
+                    const partial = open.assistantPartial.trim();
+                    finishExchange({
+                        turnId: syntheticTurnId(),
+                        user,
+                        assistant: partial ? `${partial} ${callEndedNote}` : noReplyNote,
+                        interrupted: partial.length > 0,
+                        callEnded: true,
+                        events: open.events,
+                        startedAt: open.startedAt,
+                    });
+                }
+                else if (open.assistantPartial.trim()) {
+                    stats.dropped += 1;
+                }
+                open = newExchange();
+            }
+            let timedOut = false;
+            await Promise.race([
+                queue,
+                wait(Math.max(0, (deadline ?? now()) - now())).then(() => {
+                    timedOut = true;
+                }),
+            ]);
+            if (timedOut && stats.pending > 0) {
+                logger.warn("voice.turnRecorder.flushBudgetExceeded", {
+                    pending: stats.pending,
+                    budgetMs,
+                });
+            }
+            return { ...stats };
+        },
+        stats() {
+            return { ...stats };
+        },
+    };
+}

package/dist/voice/livekit/voiceAgent.js

@@ -19,6 +19,7 @@ import { logger } from "../../utils/logger.js";
 import { createVoiceBrain } from "./brain.js";
 import { resolveBrainDefaults, resolveEouTurnDetection } from "./config.js";
 import { attachEventBridge } from "./eventBridge.js";
+import { VOICE_MEMORY_SECRET_ENV, verifyVoiceMemoryBinding, } from "./memoryBinding.js";
 const DEFAULT_CONVERSATION_PREFIX = "voice";
 // Turn-end timing defaults (approach A: silence tuning). Longer silence +
 // endpointing floor so natural mid-sentence pauses ("...and so, [pause] um")
@@ -40,15 +41,47 @@ function latestUserText(chatCtx) {
     }
     return undefined;
 }
+/**
+ * Who a call belongs to and which conversation it appends to.
+ *
+ * A verified memory binding names both. Without one (no `memory` option, or a
+ * binding that does not verify) the call keeps the static identity and the
+ * room-derived conversation id, exactly as it always did.
+ */
+function resolveCallIdentity(ctx, config, conversationPrefix) {
+    const fallbackId = `${conversationPrefix}-${ctx.room.name ?? ctx.job.id}`;
+    if (!config.memory) {
+        return { conversationId: fallbackId, bound: false };
+    }
+    const verdict = verifyVoiceMemoryBinding({
+        roomMetadata: ctx.room.metadata,
+        roomName: ctx.room.name,
+        secret: config.memory.secret ?? process.env[VOICE_MEMORY_SECRET_ENV],
+    });
+    if (!verdict.ok) {
+        logger.warn("voice.agent.memoryBindingRejected", {
+            reason: verdict.reason,
+            action: "continuing-with-static-identity",
+        });
+        return { conversationId: fallbackId, bound: false };
+    }
+    return {
+        conversationId: verdict.sessionId,
+        userId: verdict.userId,
+        bound: true,
+    };
+}
 /**
  * Build a text stream for a single turn from the brain, abortable on cancel.
  * When the framework cancels the stream (barge-in), the brain's turn is aborted.
  */
-function brainTurnStream(brain, transcript, conversationId, onAbortedBeforeOutput) {
+function brainTurnStream(brain, transcript, call, onAbortedBeforeOutput) {
     const controller = new AbortController();
     const generator = brain.streamReply({
         transcript,
-        conversationId,
+        conversationId: call.conversationId,
+        ...(call.userId !== undefined ? { userId: call.userId } : {}),
+        ...(call.ephemeral ? { ephemeral: true } : {}),
         signal: controller.signal,
     });
     const iterator = generator[Symbol.asyncIterator]();
@@ -217,7 +250,7 @@ export function defineVoiceAgent(config) {
             maxTokens: config.maxTokens,
             userId: config.userId,
         });
-        const conversationId = `${conversationPrefix}-${ctx.room.name ?? ctx.job.id}`;
+        const callIdentity = resolveCallIdentity(ctx, config, conversationPrefix);
         const { voice, llm } = await import("@livekit/agents");
         const [vad, stt, tts, eouTurnDetector] = await Promise.all([
             loadVad(config.vad),
@@ -284,7 +317,7 @@ export function defineVoiceAgent(config) {
                         ms: Date.now() - userStoppedSpeakingAt,
                     });
                 }
-                return brainTurnStream(brain, promptText, conversationId, () => {
+                return brainTurnStream(brain, promptText, callIdentity, () => {
                     // Interrupted before producing any reply → carry this turn's text
                     // forward; the next turn merges it (prompt + UI).
                     pendingPrefix = promptText;
@@ -402,7 +435,12 @@ export function defineVoiceAgent(config) {
         });
         await session.start({ agent, room: ctx.room });
         if (config.greeting !== undefined && config.greeting.trim().length > 0) {
-            const greetingStream = brainTurnStream(brain, config.greeting, conversationId);
+            const greetingStream = brainTurnStream(brain, config.greeting, {
+                ...callIdentity,
+                // The greeting is an instruction to the agent; on a bound call it must
+                // not be filed in the person's conversation as something they said.
+                ephemeral: callIdentity.bound,
+            });
             session.say(greetingStream, {
                 addToChatCtx: true,
                 allowInterruptions: true,