@juspay/neurolink 12.49.20 → 12.50.0

package/dist/voice/livekit/turnRecorder.js

@@ -0,0 +1,362 @@
+/**
+ * Turns a live voice call's event stream into stored conversation turns.
+ *
+ * A realtime voice model reports a call as separate, loosely ordered events:
+ * final user transcripts, assistant text deltas, tool calls, and the added
+ * assistant message that closes a reply. This recorder groups them into
+ * exchanges and hands each finished exchange to a `write` function (normally
+ * `NeuroLink.recordConversationTurn`).
+ *
+ * It knows nothing about LiveKit, so it can be driven by a synthetic event
+ * stream in tests. The rules it applies:
+ *
+ *  - An exchange is the user's final transcript(s) plus the assistant message
+ *    that answers them. The assistant message's id is the exchange's `turnId`,
+ *    which makes a retried write idempotent.
+ *  - Settle rule. A user's final transcript can arrive after the assistant has
+ *    started, or even finished, replying. A closed reply with no user text is
+ *    held for `settleMs`; a transcript arriving in that window completes it.
+ *    A reply still without user text after the window (the opening greeting,
+ *    typically) is dropped, because a stored turn needs both sides.
+ *  - A new user transcript arriving while the previous reply is still being
+ *    spoken, and after that exchange already had user text, closes the
+ *    previous exchange as interrupted (barge-in) and starts a new one.
+ *  - `flush` is for shutdown. It closes the open exchange with what was said
+ *    so far, marks it interrupted and call-ended, and appends a short note so
+ *    a later reader knows the reply is incomplete. A user turn with no reply
+ *    gets the note alone. The flush waits for pending writes only up to a
+ *    budget, because the worker process is reaped about a second after the
+ *    shutdown callbacks start.
+ *  - Writes run strictly one at a time, in order. A failed write is retried;
+ *    after the last retry it is reported through `onPermanentFailure` and the
+ *    recorder carries on — a storage problem never stops a call.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import { randomUUID } from "node:crypto";
+import { logger } from "../../utils/logger.js";
+const DEFAULT_SETTLE_MS = 1500;
+const DEFAULT_RETRIES = 2;
+const DEFAULT_RETRY_DELAY_MS = 250;
+const DEFAULT_CALL_ENDED_NOTE = "[The call ended before this reply finished.]";
+const DEFAULT_NO_REPLY_NOTE = "[The call ended before a reply.]";
+/** Tool arguments and results are stored with the turn; keep one entry's text bounded. */
+const MAX_TOOL_TEXT_CHARS = 4000;
+function boundedText(value) {
+    return value.length > MAX_TOOL_TEXT_CHARS
+        ? `${value.slice(0, MAX_TOOL_TEXT_CHARS)}…[truncated]`
+        : value;
+}
+function boundedPayload(value) {
+    if (typeof value === "string") {
+        return boundedText(value);
+    }
+    try {
+        const text = JSON.stringify(value);
+        return text !== undefined && text.length > MAX_TOOL_TEXT_CHARS
+            ? boundedText(text)
+            : value;
+    }
+    catch {
+        return "[unserializable]";
+    }
+}
+function wait(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+/**
+ * Store one exchange, retrying with a growing delay. After the last attempt (or
+ * once shutdown has run out of time) the failure is counted, logged and handed
+ * to `onPermanentFailure`; nothing here throws, so a storage problem never
+ * stops a call.
+ */
+async function writeWithRetry(record, context) {
+    const { options, stats, retries, retryDelayMs, isOutOfTime } = context;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt += 1) {
+        try {
+            await options.write(record);
+            stats.written += 1;
+            options.onRecorded?.(record);
+            return;
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt === retries || isOutOfTime()) {
+                break;
+            }
+            await wait(retryDelayMs * (attempt + 1));
+        }
+    }
+    stats.failed += 1;
+    logger.error("voice.turnRecorder.writeFailed", {
+        turnId: record.turnId,
+        error: lastError instanceof Error ? lastError.message : String(lastError),
+    });
+    try {
+        options.onPermanentFailure?.(record, lastError);
+    }
+    catch {
+        /* a failure sink must never break the call */
+    }
+}
+/** Create a recorder for one call. */
+export function createVoiceTurnRecorder(options) {
+    const settleMs = options.settleMs ?? DEFAULT_SETTLE_MS;
+    const retries = options.retries ?? DEFAULT_RETRIES;
+    const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+    const callEndedNote = options.callEndedNote ?? DEFAULT_CALL_ENDED_NOTE;
+    const noReplyNote = options.noReplyNote ?? DEFAULT_NO_REPLY_NOTE;
+    const now = options.now ?? Date.now;
+    const callId = options.callId ?? randomUUID();
+    const stats = {
+        written: 0,
+        failed: 0,
+        dropped: 0,
+        pending: 0,
+    };
+    let open = newExchange();
+    // Every reply id that has been written or queued, so a reply delivered a
+    // second time can never be paired with the next question.
+    const seenReplyIds = new Set();
+    let greetingPending = false;
+    const awaitingUser = [];
+    let queue = Promise.resolve();
+    let eventSeq = 0;
+    let syntheticTurns = 0;
+    let closing = false;
+    let deadline;
+    function newExchange() {
+        return {
+            userSegments: [],
+            assistantPartial: "",
+            assistantStarted: false,
+            events: [],
+            startedAt: now(),
+        };
+    }
+    function nextEvent(type, fields) {
+        const event = {
+            type,
+            seq: eventSeq,
+            timestamp: now(),
+            ...fields,
+        };
+        eventSeq += 1;
+        return event;
+    }
+    function enqueue(record) {
+        stats.pending += 1;
+        queue = queue.then(async () => {
+            try {
+                await writeWithRetry(record, {
+                    options,
+                    stats,
+                    retries,
+                    retryDelayMs,
+                    isOutOfTime: () => closing && deadline !== undefined && now() >= deadline,
+                });
+            }
+            finally {
+                stats.pending -= 1;
+            }
+        });
+    }
+    function finishExchange(exchange) {
+        enqueue({
+            turnId: exchange.turnId,
+            userMessage: exchange.user,
+            aiResponse: exchange.assistant,
+            interrupted: exchange.interrupted,
+            callEnded: exchange.callEnded,
+            startTime: new Date(exchange.startedAt),
+            ...(exchange.events.length > 0 ? { events: exchange.events } : {}),
+        });
+    }
+    function syntheticTurnId() {
+        syntheticTurns += 1;
+        return `call-${callId}-${syntheticTurns}`;
+    }
+    function dropClosed(exchange) {
+        clearTimeout(exchange.timer);
+        stats.dropped += 1;
+        logger.debug("voice.turnRecorder.assistantOnlyDropped", {
+            turnId: exchange.turnId,
+        });
+    }
+    return {
+        userTranscript(segment) {
+            const text = segment.text.trim();
+            if (closing || !segment.final || !text) {
+                return;
+            }
+            const waiting = awaitingUser.shift();
+            if (waiting) {
+                clearTimeout(waiting.timer);
+                finishExchange({
+                    turnId: waiting.turnId,
+                    user: text,
+                    assistant: waiting.assistantText,
+                    interrupted: waiting.interrupted,
+                    callEnded: false,
+                    events: waiting.events,
+                    startedAt: waiting.startedAt,
+                });
+                return;
+            }
+            if (open.assistantStarted && open.userSegments.length > 0) {
+                // The user spoke again while the previous reply was still going: close
+                // that exchange as cut off, and start a new one with this transcript.
+                finishExchange({
+                    turnId: syntheticTurnId(),
+                    user: open.userSegments.join(" "),
+                    assistant: open.assistantPartial.trim() || noReplyNote,
+                    interrupted: true,
+                    callEnded: false,
+                    events: open.events,
+                    startedAt: open.startedAt,
+                });
+                open = newExchange();
+            }
+            open.userSegments.push(text);
+        },
+        assistantDelta(delta) {
+            if (closing || !delta) {
+                return;
+            }
+            open.assistantStarted = true;
+            open.assistantPartial += delta;
+        },
+        assistantMessage(message) {
+            if (closing) {
+                return;
+            }
+            const text = message.text.trim() || open.assistantPartial.trim();
+            if (!text) {
+                // An item with no spoken text (a tool-only message) closes nothing.
+                return;
+            }
+            if (seenReplyIds.has(message.id)) {
+                // The same reply delivered again, whether it is still waiting for its
+                // user text or was already written.
+                return;
+            }
+            seenReplyIds.add(message.id);
+            if (greetingPending) {
+                // The agent's opening line is an instruction, not an answer: it is not
+                // a turn, and it must never be paired with the user's first question
+                // (they may speak over it).
+                greetingPending = false;
+                // Whatever the user has already said stays: it belongs to the reply
+                // that follows, not to the greeting.
+                open.assistantPartial = "";
+                open.assistantStarted = false;
+                return;
+            }
+            const exchange = open;
+            open = newExchange();
+            const interrupted = message.interrupted === true;
+            if (exchange.userSegments.length > 0) {
+                finishExchange({
+                    turnId: message.id,
+                    user: exchange.userSegments.join(" "),
+                    assistant: text,
+                    interrupted,
+                    callEnded: false,
+                    events: exchange.events,
+                    startedAt: exchange.startedAt,
+                });
+                return;
+            }
+            const closed = {
+                turnId: message.id,
+                assistantText: text,
+                interrupted,
+                events: exchange.events,
+                startedAt: exchange.startedAt,
+                timer: setTimeout(() => {
+                    const index = awaitingUser.indexOf(closed);
+                    if (index >= 0) {
+                        awaitingUser.splice(index, 1);
+                        dropClosed(closed);
+                    }
+                }, settleMs),
+            };
+            closed.timer.unref?.();
+            awaitingUser.push(closed);
+        },
+        expectGreeting() {
+            greetingPending = true;
+        },
+        toolStart(call) {
+            if (closing) {
+                return;
+            }
+            open.events.push(nextEvent("tool:start", {
+                tool: call.name,
+                toolName: call.name,
+                ...(call.args !== undefined
+                    ? { args: boundedPayload(call.args) }
+                    : {}),
+            }));
+        },
+        toolEnd(call) {
+            if (closing) {
+                return;
+            }
+            open.events.push(nextEvent("tool:end", {
+                tool: call.name,
+                toolName: call.name,
+                success: call.error === undefined,
+                ...(call.error !== undefined
+                    ? { error: boundedText(call.error) }
+                    : { result: boundedPayload(call.result) }),
+            }));
+        },
+        async flush(budgetMs) {
+            if (!closing) {
+                closing = true;
+                deadline = now() + budgetMs;
+                for (const waiting of awaitingUser.splice(0)) {
+                    dropClosed(waiting);
+                }
+                const user = open.userSegments.join(" ");
+                if (user) {
+                    const partial = open.assistantPartial.trim();
+                    finishExchange({
+                        turnId: syntheticTurnId(),
+                        user,
+                        assistant: partial ? `${partial} ${callEndedNote}` : noReplyNote,
+                        interrupted: partial.length > 0,
+                        callEnded: true,
+                        events: open.events,
+                        startedAt: open.startedAt,
+                    });
+                }
+                else if (open.assistantPartial.trim()) {
+                    stats.dropped += 1;
+                }
+                open = newExchange();
+            }
+            let timedOut = false;
+            await Promise.race([
+                queue,
+                wait(Math.max(0, (deadline ?? now()) - now())).then(() => {
+                    timedOut = true;
+                }),
+            ]);
+            if (timedOut && stats.pending > 0) {
+                logger.warn("voice.turnRecorder.flushBudgetExceeded", {
+                    pending: stats.pending,
+                    budgetMs,
+                });
+            }
+            return { ...stats };
+        },
+        stats() {
+            return { ...stats };
+        },
+    };
+}

package/dist/voice/livekit/voiceAgent.js

@@ -19,6 +19,7 @@ import { logger } from "../../utils/logger.js";
 import { createVoiceBrain } from "./brain.js";
 import { resolveBrainDefaults, resolveEouTurnDetection } from "./config.js";
 import { attachEventBridge } from "./eventBridge.js";
+import { VOICE_MEMORY_SECRET_ENV, verifyVoiceMemoryBinding, } from "./memoryBinding.js";
 const DEFAULT_CONVERSATION_PREFIX = "voice";
 // Turn-end timing defaults (approach A: silence tuning). Longer silence +
 // endpointing floor so natural mid-sentence pauses ("...and so, [pause] um")
@@ -40,15 +41,47 @@ function latestUserText(chatCtx) {
     }
     return undefined;
 }
+/**
+ * Who a call belongs to and which conversation it appends to.
+ *
+ * A verified memory binding names both. Without one (no `memory` option, or a
+ * binding that does not verify) the call keeps the static identity and the
+ * room-derived conversation id, exactly as it always did.
+ */
+function resolveCallIdentity(ctx, config, conversationPrefix) {
+    const fallbackId = `${conversationPrefix}-${ctx.room.name ?? ctx.job.id}`;
+    if (!config.memory) {
+        return { conversationId: fallbackId, bound: false };
+    }
+    const verdict = verifyVoiceMemoryBinding({
+        roomMetadata: ctx.room.metadata,
+        roomName: ctx.room.name,
+        secret: config.memory.secret ?? process.env[VOICE_MEMORY_SECRET_ENV],
+    });
+    if (!verdict.ok) {
+        logger.warn("voice.agent.memoryBindingRejected", {
+            reason: verdict.reason,
+            action: "continuing-with-static-identity",
+        });
+        return { conversationId: fallbackId, bound: false };
+    }
+    return {
+        conversationId: verdict.sessionId,
+        userId: verdict.userId,
+        bound: true,
+    };
+}
 /**
  * Build a text stream for a single turn from the brain, abortable on cancel.
  * When the framework cancels the stream (barge-in), the brain's turn is aborted.
  */
-function brainTurnStream(brain, transcript, conversationId, onAbortedBeforeOutput) {
+function brainTurnStream(brain, transcript, call, onAbortedBeforeOutput) {
     const controller = new AbortController();
     const generator = brain.streamReply({
         transcript,
-        conversationId,
+        conversationId: call.conversationId,
+        ...(call.userId !== undefined ? { userId: call.userId } : {}),
+        ...(call.ephemeral ? { ephemeral: true } : {}),
         signal: controller.signal,
     });
     const iterator = generator[Symbol.asyncIterator]();
@@ -217,7 +250,7 @@ export function defineVoiceAgent(config) {
             maxTokens: config.maxTokens,
             userId: config.userId,
         });
-        const conversationId = `${conversationPrefix}-${ctx.room.name ?? ctx.job.id}`;
+        const callIdentity = resolveCallIdentity(ctx, config, conversationPrefix);
         const { voice, llm } = await import("@livekit/agents");
         const [vad, stt, tts, eouTurnDetector] = await Promise.all([
             loadVad(config.vad),
@@ -284,7 +317,7 @@ export function defineVoiceAgent(config) {
                         ms: Date.now() - userStoppedSpeakingAt,
                     });
                 }
-                return brainTurnStream(brain, promptText, conversationId, () => {
+                return brainTurnStream(brain, promptText, callIdentity, () => {
                     // Interrupted before producing any reply → carry this turn's text
                     // forward; the next turn merges it (prompt + UI).
                     pendingPrefix = promptText;
@@ -402,7 +435,12 @@ export function defineVoiceAgent(config) {
         });
         await session.start({ agent, room: ctx.room });
         if (config.greeting !== undefined && config.greeting.trim().length > 0) {
-            const greetingStream = brainTurnStream(brain, config.greeting, conversationId);
+            const greetingStream = brainTurnStream(brain, config.greeting, {
+                ...callIdentity,
+                // The greeting is an instruction to the agent; on a bound call it must
+                // not be filed in the person's conversation as something they said.
+                ephemeral: callIdentity.bound,
+            });
             session.say(greetingStream, {
                 addToChatCtx: true,
                 allowInterruptions: true,

package/docs-site/static/search-index.json

@@ -3668,6 +3668,7 @@
 {"objectID":"e702f5ae42413e90dc937d8b3216485da56c3baaef4130e033bd9bdc199e74a2","title":"Language & multilingual speech","url":"/docs/features/livekit-voice-agent#language-multilingual-speech","content":"The language field on stt is a soft hint: it biases recognition toward a\nlanguage without locking to it, so a user can switch languages mid-call and still\nbe transcribed correctly.\nOmit language for full auto-detection.\nThe hint only biases the first guess; it never forces the hinted language. (A\n  strict lock causes the realtime stream to stall on other-language audio, so the\n  integration intentionally keeps the hint soft.)","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Language & multilingual speech","lvl3":""}},
 {"objectID":"804a64dd17c69cb8d4af469682d01d12f00bec2d6869a5faef30865da99f176d","title":"Speech provider selection","url":"/docs/features/livekit-voice-agent#speech-provider-selection","content":"STT and TTS plugins are chosen per agent and configured by environment credentials.\nSTT: soniox, deepgram. TTS: cartesia, elevenlabs.\nOnly set voice / model if your account supports them; otherwise omit those\n  fields to use the plugin's own defaults.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Speech provider selection","lvl3":""}},
 {"objectID":"5589375c3b5b8ee71271f32386ef6b0f878bc88f7097a13ee7d8ef03b694655e","title":"Conversation Memory","url":"/docs/features/livekit-voice-agent#conversation-memory","content":"The agent remembers earlier turns automatically when the NeuroLink instance you\nbuild inside createNeuroLink has conversation memory enabled. History is the\nagent's source of truth — LiveKit's own transcript context is not used for\ngeneration.\n\nHow it behaves:\nKeyed per call. Each room/call is an isolated conversation; the id is\n  derived from the room name. Override the prefix with conversationIdPrefix\n  (default \"voice\").\nIn-memory by default; Redis for persistence. Set REDIS_URL to use a\n  shared store that survives worker restarts and is shared across worker\n  replicas — important because each call runs in its own job process.\nWorks across turns within the session. The user can say \"my name is Alex\"\n  and later ask \"what's my name?\" and the agent recalls it.\n\nMemory persists only when the instance is configured with\nconversationMemory.enabled. Without it, each turn is independent.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Conversation Memory","lvl3":""}},
+{"objectID":"e6ef692117fe2ad968f2bb5bd1692e50783a069a189d3dbfd6ac49f71ef84c73","title":"Remembering a realtime call in the user's own conversation","url":"/docs/features/livekit-voice-agent#remembering-a-realtime-call-in-the-users-own-conversation","content":"A speech-to-speech call (defineRealtimeVoiceAgent) never goes through\nstream() or generate(), so by default nothing it says is stored. Opt in with\nmemory and the call's turns are written, through\nNeuroLink.recordConversationTurn, to the same conversation memory typed chat\nuses, appended to the conversation the person already has open. The model is\nalso seeded with that conversation's recent history when the call starts, so it\nknows what was said before.\n\nWhose conversation is never taken from the client. The host that creates\nthe room signs a binding and puts it in the room metadata next to authToken\nand mcpContext. The worker stores nothing unless it verifies.\nThe signature is HMAC-SHA256 over [\"neurolink.voice.memory.v1\", room,\nuserId, sessionId, exp] (a JSON array, so no choice of values can make two\n  bindings collide), base64url-encoded. It covers the room name, so a binding\n  copied into another room does not verify. exp is Unix seconds; the default\n  lifetime is two hours, the cap 24 hours.\nThe shared secret is at least 16 characters, read from\n  NEUROLINK_VOICE_MEMORY_SECRET unless memory.secret is set on the worker.\nsessionId matches ^[A-Za-z0-9_-]{1,128}$ (no :, which is part of the\n  stored key). userId is 1 to 256 characters with no whitespace or control\n  characters; : is allowed because chat user ids contain it.\nFail closed. No secret, no binding, a malformed or expired binding, or a\n  bad signature: the call runs normally, nothing is stored, and a warning\n  names the reason (never the ids).\nA client may name the conversation to append to. The host accepts the name\n  only when neurolink.conversationExists({ sessionId, userId }) is true for\n  the verified user. Otherwise it mints a new voice-<uuid>.\n\nWhat is stored. Each finished exchange becomes a user message and an\nassistant message with the ids ${turnId}:user and ${turnId}:assistant, where\nturnId is the realtime session's assistant message id, and\nmetadata.modality: \"voice\".","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Remembering a realtime call in the user's own conversation","lvl3":""}},
 {"objectID":"7ffe7e32a7c2baea6c491d743617448eb70eeb4afe439960050efdc3ead18c48","title":"Implementation Plan","url":"/docs/features/livekit-voice-agent#implementation-plan","content":"The integration is built and validated in phases. Each phase is independently testable.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Implementation Plan","lvl3":""}},
 {"objectID":"d4c9885eea3272e250fa1a0dbc1af9503b1a24f0217a82dd0c30d93bf6c5df31","title":"Phase 0 — Console-mode spike (no infrastructure)","url":"/docs/features/livekit-voice-agent#phase-0-console-mode-spike-no-infrastructure","content":"Build a minimal agent (Silero VAD + Deepgram STT + ElevenLabs TTS + llmNode → neurolink.stream()) and run it in console mode using the host machine's mic/speakers. Validates the NeuroLink brain loop, conversationId history, and a tool call — with no LiveKit server and no browser. Requires only STT/TTS and LLM credentials.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Phase 0 — Console-mode spike (no infrastructure)","lvl3":""}},
 {"objectID":"9dddf98f952af643b5357bdefbcb5554a3ff2d9c33a16b0c432243fd68db5569","title":"Phase 1 — NeuroLink LiveKit module","url":"/docs/features/livekit-voice-agent#phase-1-neurolink-livekit-module","content":"Implement brain.ts, voiceAgent.ts, voiceAgentWorker.ts, tokens.ts, config.ts; add the @juspay/neurolink/livekit export and optional/peer dependencies. The worker factory accepts an external NeuroLink instance so a host application's registered tools are available. Wire abort-on-interrupt. Verify build, type-check, and lint.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Phase 1 — NeuroLink LiveKit module","lvl3":""}},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.49.20",
+  "version": "12.50.0",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -113,6 +113,7 @@
     "test:adjust-body-after-400": "pnpm exec tsx test/continuous-test-suite-adjust-body-after-400.ts",
     "test:zod3-schema-native-path": "pnpm exec tsx test/continuous-test-suite-zod3-schema-native-path.ts",
     "test:forced-tool-declared": "pnpm exec tsx test/continuous-test-suite-forced-tool-declared.ts",
+    "test:voice-turn-memory": "pnpm exec tsx test/continuous-test-suite-voice-turn-memory.ts",
     "test:error-classification-e2e": "pnpm exec tsx test/continuous-test-suite-error-classification-e2e.ts",
     "test:error-classifier-contract": "pnpm exec tsx test/continuous-test-suite-error-classifier-contract.ts",
     "test:bedrock-inference-profile": "pnpm exec tsx test/continuous-test-suite-bedrock-inference-profile.ts",