@juspay/neurolink 12.49.19 → 12.50.0

package/dist/voice/livekit/memoryBinding.js

@@ -0,0 +1,152 @@
+/**
+ * Signed memory binding for a voice call.
+ *
+ * A voice worker persists a call's turns to a conversation, so it has to know
+ * WHOSE conversation and WHICH one — and it must not take either from anything
+ * a participant can influence. The manager that creates the room (for example
+ * a `/start` endpoint) already holds the verified session, so it signs a small
+ * binding and writes it into the room metadata next to `authToken` and
+ * `mcpContext`:
+ *
+ *   base64(JSON({ authToken, mcpContext, memory: { userId, sessionId, exp, sig } }))
+ *
+ * The worker verifies the signature with the same shared secret. The signature
+ * covers the room name, so a binding copied into another room does not verify.
+ * Room metadata is readable by participants, so this does not hide the ids; it
+ * stops anyone without the secret from choosing them.
+ *
+ * Everything here fails closed. A missing secret, a missing or malformed
+ * binding, a bad signature or an expired binding all yield `{ ok: false }`, and
+ * the caller carries on with the call, storing nothing.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import { createHmac, timingSafeEqual } from "node:crypto";
+import { Buffer } from "node:buffer";
+import { z } from "zod";
+/** Domain-separation label that starts every signed message. */
+export const VOICE_MEMORY_BINDING_VERSION = "neurolink.voice.memory.v1";
+/** Environment variable the worker reads the shared secret from when none is passed in. */
+export const VOICE_MEMORY_SECRET_ENV = "NEUROLINK_VOICE_MEMORY_SECRET";
+/** Longest expiry, counted from now, that a worker accepts (24 hours). */
+export const VOICE_MEMORY_MAX_TTL_SECONDS = 24 * 60 * 60;
+/** A secret shorter than this is treated as absent. */
+const MIN_SECRET_LENGTH = 16;
+const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;
+// Whitespace and control characters are refused; ":" is allowed because the
+// chat user id is itself composed with it (customer:merchant:shop).
+// eslint-disable-next-line no-control-regex
+const USER_ID_PATTERN = /^[^\s\u0000-\u001f\u007f]{1,256}$/u;
+/** A conversation id a voice call may append to: letters, digits, "_" and "-", up to 128, no ":". */
+export function isValidVoiceSessionId(value) {
+    return typeof value === "string" && SESSION_ID_PATTERN.test(value);
+}
+/** A user id a voice call may be bound to: 1 to 256 characters, no whitespace or control characters. */
+export function isValidVoiceUserId(value) {
+    return typeof value === "string" && USER_ID_PATTERN.test(value);
+}
+function usableSecret(secret) {
+    return typeof secret === "string" && secret.length >= MIN_SECRET_LENGTH;
+}
+/**
+ * What is signed. A JSON array, not a delimiter-joined string, so no choice of
+ * field values can make two different bindings produce the same message.
+ */
+function signedMessage(room, userId, sessionId, exp) {
+    return JSON.stringify([
+        VOICE_MEMORY_BINDING_VERSION,
+        room,
+        userId,
+        sessionId,
+        exp,
+    ]);
+}
+function signature(secret, message) {
+    return createHmac("sha256", secret).update(message).digest();
+}
+/**
+ * Sign a binding for a room. Throws on an unusable secret or invalid ids: the
+ * signer is the trusted side, and a binding it cannot sign correctly is a bug
+ * to surface there, not something to paper over.
+ */
+export function signVoiceMemoryBinding(input) {
+    const { room, userId, sessionId, secret } = input;
+    if (!room) {
+        throw new Error("A voice memory binding needs the room name");
+    }
+    if (!usableSecret(secret)) {
+        throw new Error(`The voice memory secret must be at least ${MIN_SECRET_LENGTH} characters`);
+    }
+    if (!isValidVoiceUserId(userId)) {
+        throw new Error("Invalid voice memory userId");
+    }
+    if (!isValidVoiceSessionId(sessionId)) {
+        throw new Error("Invalid voice memory sessionId");
+    }
+    const now = input.now ?? Math.floor(Date.now() / 1000);
+    const ttl = input.ttlSeconds ?? 2 * 60 * 60;
+    if (!Number.isInteger(ttl) ||
+        ttl <= 0 ||
+        ttl > VOICE_MEMORY_MAX_TTL_SECONDS) {
+        throw new Error(`ttlSeconds must be a whole number between 1 and ${VOICE_MEMORY_MAX_TTL_SECONDS}`);
+    }
+    const exp = now + ttl;
+    const sig = signature(secret, signedMessage(room, userId, sessionId, exp)).toString("base64url");
+    return { userId, sessionId, exp, sig };
+}
+const bindingSchema = z.object({
+    userId: z.string(),
+    sessionId: z.string(),
+    exp: z.number().int(),
+    sig: z.string(),
+});
+const roomMetadataSchema = z.object({ memory: z.unknown().optional() });
+function decodeMetadata(encoded) {
+    try {
+        return JSON.parse(Buffer.from(encoded, "base64").toString("utf-8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Read and verify the memory binding in a room's metadata.
+ *
+ * Never throws and never logs ids. `reason` says why a binding was rejected so
+ * the caller can log it.
+ */
+export function verifyVoiceMemoryBinding(input) {
+    const { roomMetadata, roomName, secret } = input;
+    if (!usableSecret(secret)) {
+        return { ok: false, reason: "no-secret" };
+    }
+    if (!roomName || !roomMetadata) {
+        return { ok: false, reason: "no-binding" };
+    }
+    const metadata = roomMetadataSchema.safeParse(decodeMetadata(roomMetadata));
+    if (!metadata.success || metadata.data.memory === undefined) {
+        return { ok: false, reason: "no-binding" };
+    }
+    const parsed = bindingSchema.safeParse(metadata.data.memory);
+    if (!parsed.success) {
+        return { ok: false, reason: "malformed" };
+    }
+    const { userId, sessionId, exp, sig } = parsed.data;
+    if (!isValidVoiceUserId(userId) || !isValidVoiceSessionId(sessionId)) {
+        return { ok: false, reason: "malformed" };
+    }
+    const expected = signature(secret, signedMessage(roomName, userId, sessionId, exp));
+    const presented = Buffer.from(sig, "base64url");
+    if (presented.length !== expected.length ||
+        !timingSafeEqual(presented, expected)) {
+        return { ok: false, reason: "bad-signature" };
+    }
+    const now = input.now ?? Math.floor(Date.now() / 1000);
+    if (exp <= now) {
+        return { ok: false, reason: "expired" };
+    }
+    if (exp - now > VOICE_MEMORY_MAX_TTL_SECONDS) {
+        return { ok: false, reason: "expiry-too-far" };
+    }
+    return { ok: true, userId, sessionId, exp };
+}

package/dist/voice/livekit/realtimeMcpTools.js

@@ -94,7 +94,18 @@ function registerToolAliases(toolContext, mcpToolName, handler) {
  * filtering is needed.
  */
 export async function buildRealtimeMcpTools(params) {
-    const { mcpUrl, authToken, xContext, publishEvent, requestConfirmation, toolTimeoutMs = DEFAULT_TOOL_TIMEOUT_MS, } = params;
+    const { mcpUrl, authToken, xContext, publishEvent, requestConfirmation, toolTimeoutMs = DEFAULT_TOOL_TIMEOUT_MS, onToolStart, onToolEnd, } = params;
+    // Observers feed the turn recorder; a fault in one must never reach the tool.
+    const observe = (notify) => {
+        try {
+            notify();
+        }
+        catch (error) {
+            logger.warn("realtime.tool.observerFailed", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    };
     const { llm } = await import("@livekit/agents");
     const { Client: McpClient } = await import("@modelcontextprotocol/sdk/client/index.js");
     const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
@@ -147,6 +158,7 @@ export async function buildRealtimeMcpTools(params) {
                         args: args ?? {},
                     });
                     publishEvent("tool-start", { name: mcpTool.name });
+                    observe(() => onToolStart?.({ name: mcpTool.name, args: args ?? {} }));
                     const startedAt = Date.now();
                     try {
                         // Third argument is RequestOptions; the SDK aborts the in-flight
@@ -167,6 +179,12 @@ export async function buildRealtimeMcpTools(params) {
                         // Forward the RAW result so the client can extract a
                         // chart/UIComponent and clear the in-progress indicator.
                         publishEvent("tool-result", { name: mcpTool.name, result });
+                        observe(() => onToolEnd?.({
+                            name: mcpTool.name,
+                            ...(result.isError === true
+                                ? { error: text }
+                                : { result: text }),
+                        }));
                         return text;
                     }
                     catch (error) {
@@ -176,6 +194,7 @@ export async function buildRealtimeMcpTools(params) {
                             error: error instanceof Error ? error.message : String(error),
                         });
                         publishEvent("tool-result", { name: mcpTool.name, result: null });
+                        observe(() => onToolEnd?.({ name: mcpTool.name, error: String(error) }));
                         return `Tool ${mcpTool.name} failed: ${String(error)}`;
                     }
                 },

package/dist/voice/livekit/realtimeMemory.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Per-call memory for the realtime (speech-to-speech) voice agent.
+ *
+ * Decides, once per call and before anything else is wired, whether this call
+ * may be remembered: the room metadata must carry a memory binding that
+ * verifies against the shared secret and the actual room name. If it does, a
+ * NeuroLink instance is built, the conversation's recent history is read for
+ * seeding, and a turn recorder is created that writes each finished exchange
+ * through `recordConversationTurn`. If it does not, the call proceeds exactly
+ * as it would without memory, and a warning says why (never the ids).
+ *
+ * The shutdown flush is registered here, before the job lifecycle installs its
+ * process reaper. LiveKit starts every shutdown callback in registration order
+ * and the reaper exits the process about a second later, so the flush gets a
+ * budget below that second and starts first.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import type { JobContext } from "@livekit/agents";
+import type { RealtimeCallMemory, RealtimeVoiceMemoryConfig } from "../../types/index.js";
+/** Verify the call's binding and prepare its recorder; `undefined` means "no memory for this call". */
+export declare function setUpRealtimeCallMemory(ctx: Pick<JobContext, "room" | "addShutdownCallback">, config: RealtimeVoiceMemoryConfig | undefined): Promise<RealtimeCallMemory | undefined>;

package/dist/voice/livekit/realtimeMemory.js

@@ -0,0 +1,111 @@
+/**
+ * Per-call memory for the realtime (speech-to-speech) voice agent.
+ *
+ * Decides, once per call and before anything else is wired, whether this call
+ * may be remembered: the room metadata must carry a memory binding that
+ * verifies against the shared secret and the actual room name. If it does, a
+ * NeuroLink instance is built, the conversation's recent history is read for
+ * seeding, and a turn recorder is created that writes each finished exchange
+ * through `recordConversationTurn`. If it does not, the call proceeds exactly
+ * as it would without memory, and a warning says why (never the ids).
+ *
+ * The shutdown flush is registered here, before the job lifecycle installs its
+ * process reaper. LiveKit starts every shutdown callback in registration order
+ * and the reaper exits the process about a second later, so the flush gets a
+ * budget below that second and starts first.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import { logger } from "../../utils/logger.js";
+import { VOICE_MEMORY_SECRET_ENV, verifyVoiceMemoryBinding, } from "./memoryBinding.js";
+import { createVoiceTurnRecorder } from "./turnRecorder.js";
+const DEFAULT_FLUSH_BUDGET_MS = 800;
+/** Verify the call's binding and prepare its recorder; `undefined` means "no memory for this call". */
+export async function setUpRealtimeCallMemory(ctx, config) {
+    if (!config) {
+        return undefined;
+    }
+    const verdict = verifyVoiceMemoryBinding({
+        roomMetadata: ctx.room.metadata,
+        roomName: ctx.room.name,
+        secret: config.secret ?? process.env[VOICE_MEMORY_SECRET_ENV],
+    });
+    if (!verdict.ok) {
+        logger.warn("realtime.memory.bindingRejected", {
+            reason: verdict.reason,
+            action: "continuing-without-memory",
+        });
+        return undefined;
+    }
+    const { userId, sessionId } = verdict;
+    let neurolink;
+    try {
+        neurolink = await config.createNeuroLink();
+    }
+    catch (error) {
+        logger.error("realtime.memory.setupFailed", {
+            error: error instanceof Error ? error.message : String(error),
+            action: "continuing-without-memory",
+        });
+        return undefined;
+    }
+    let publish;
+    const recorder = createVoiceTurnRecorder({
+        ...(config.settleMs !== undefined ? { settleMs: config.settleMs } : {}),
+        write: async (turn) => {
+            const result = await neurolink.recordConversationTurn({
+                sessionId,
+                userId,
+                userMessage: turn.userMessage,
+                aiResponse: turn.aiResponse,
+                modality: "voice",
+                turnId: turn.turnId,
+                interrupted: turn.interrupted,
+                callEnded: turn.callEnded,
+                startTime: turn.startTime,
+                ...(turn.events ? { events: turn.events } : {}),
+            });
+            if (!result.recorded) {
+                logger.debug("realtime.memory.turnSkipped", {
+                    reason: result.reason,
+                    turnId: turn.turnId,
+                });
+            }
+        },
+        onPermanentFailure: (turn) => {
+            publish?.("memory", { state: "not-saved", turnId: turn.turnId });
+        },
+    });
+    ctx.addShutdownCallback(async () => {
+        const stats = await recorder.flush(config.flushBudgetMs ?? DEFAULT_FLUSH_BUDGET_MS);
+        logger.info("realtime.memory.flushed", { ...stats });
+    });
+    let seed = [];
+    if (config.seed !== false && neurolink.getConversationSeed) {
+        try {
+            seed = await neurolink.getConversationSeed({
+                sessionId,
+                userId,
+                ...(config.seed?.maxTurns !== undefined
+                    ? { maxTurns: config.seed.maxTurns }
+                    : {}),
+                ...(config.seed?.maxChars !== undefined
+                    ? { maxChars: config.seed.maxChars }
+                    : {}),
+            });
+        }
+        catch (error) {
+            logger.warn("realtime.memory.seedFailed", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    logger.info("realtime.memory.enabled", { seeded: seed.length });
+    return {
+        recorder,
+        seed,
+        setPublisher: (publisher) => {
+            publish = publisher;
+        },
+    };
+}

package/dist/voice/livekit/realtimeVoiceAgent.js

@@ -19,6 +19,7 @@ import { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
 import { readCallContextFromRoom } from "./roomContext.js";
 import { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
 import { buildRealtimeMcpTools } from "./realtimeMcpTools.js";
+import { setUpRealtimeCallMemory } from "./realtimeMemory.js";
 const realtimeLogEventSchema = z.object({
     level: z.enum(["debug", "info", "warn", "error"]),
     message: z.string(),
@@ -144,6 +145,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
     const eventsTopic = config.eventsTopic;
     const controlTopic = config.controlTopic;
     const onLog = config.onLog;
+    const memoryConfig = config.memory;
     async function entry(ctx) {
         // The Gemini Live WS authenticates to Vertex via ADC; materialise it and
         // force ADC (not an API key) auth before any realtime connection.
@@ -192,6 +194,9 @@ export function defineRealtimeVoiceAgent(config = {}) {
             location: cfg.location,
             ms: sinceConnect(),
         });
+        // Decide whether this call may be remembered BEFORE the lifecycle installs
+        // its process reaper, so the shutdown flush is registered first.
+        const memory = await setUpRealtimeCallMemory(ctx, memoryConfig);
         await installRealtimeJobLifecycle(ctx, cfg);
         const { voice } = await import("@livekit/agents");
         const google = await import("@livekit/agents-plugin-google");
@@ -205,6 +210,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
         ctx.addShutdownCallback(async () => {
             bridge.dispose();
         });
+        memory?.setPublisher(bridge.publishEvent);
         let agentTools;
         if (cfg.toolsEnabled) {
             const { authToken, xContext } = readCallContextFromRoom(ctx.room.metadata);
@@ -222,6 +228,12 @@ export function defineRealtimeVoiceAgent(config = {}) {
                         xContext,
                         publishEvent: bridge.publishEvent,
                         requestConfirmation: bridge.requestConfirmation,
+                        ...(memory
+                            ? {
+                                onToolStart: (call) => memory.recorder.toolStart(call),
+                                onToolEnd: (call) => memory.recorder.toolEnd(call),
+                            }
+                            : {}),
                     });
                     agentTools = toolset.tools;
                     logger.info("realtime.mcp.enabled", {
@@ -265,9 +277,23 @@ export function defineRealtimeVoiceAgent(config = {}) {
         const session = new voice.AgentSession({
             llm: new google.realtime.RealtimeModel(modelOptions),
         });
+        // The conversation so far, so a call that appends to an open conversation
+        // starts knowing it. Empty (and omitted) when there is nothing to seed.
+        let seededChatCtx;
+        if (memory && memory.seed.length > 0) {
+            const { llm } = await import("@livekit/agents");
+            seededChatCtx = llm.ChatContext.empty();
+            for (const message of memory.seed) {
+                seededChatCtx.addMessage({
+                    role: message.role,
+                    content: message.content,
+                });
+            }
+        }
         const agent = new voice.Agent({
             instructions: cfg.systemPrompt,
             ...(agentTools ? { tools: agentTools } : {}),
+            ...(seededChatCtx ? { chatCtx: seededChatCtx } : {}),
         });
         agent.transcriptionNode = async (textStream, _modelSettings) => {
             const [forUi, forDownstream] = textStream.tee();
@@ -290,6 +316,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
                                 });
                             }
                             bridge.publishEvent("text", { delta: chunk });
+                            memory?.recorder.assistantDelta(chunk);
                         }
                     }
                 }
@@ -328,6 +355,10 @@ export function defineRealtimeVoiceAgent(config = {}) {
                 text: ev.transcript,
                 final: ev.isFinal,
             });
+            memory?.recorder.userTranscript({
+                text: ev.transcript,
+                final: ev.isFinal,
+            });
         });
         session.on(voice.AgentSessionEventTypes.AgentStateChanged, (ev) => {
             logger.debug("realtime.session.stateChanged", {
@@ -342,7 +373,14 @@ export function defineRealtimeVoiceAgent(config = {}) {
                 ev.item.role === "assistant" &&
                 ev.item.textContent) {
                 logger.debug("realtime.session.turnDone", { ms: sinceConnect() });
-                bridge.publishEvent("done", {});
+                // The id the stored messages carry, so a client can merge its live
+                // copy of the turn with history instead of showing it twice.
+                bridge.publishEvent("done", memory ? { turnId: ev.item.id } : {});
+                memory?.recorder.assistantMessage({
+                    id: ev.item.id,
+                    text: ev.item.textContent,
+                    interrupted: ev.item.interrupted,
+                });
             }
         });
         session.on(voice.AgentSessionEventTypes.Error, (ev) => {
@@ -357,6 +395,7 @@ export function defineRealtimeVoiceAgent(config = {}) {
         bridge.publishEvent("status", { state: "listening" });
         if (cfg.greeting.trim().length > 0) {
             logger.info("realtime.session.greeting", { ms: sinceConnect() });
+            memory?.recorder.expectGreeting();
             session.generateReply({ instructions: cfg.greeting });
         }
     }

package/dist/voice/livekit/turnRecorder.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Turns a live voice call's event stream into stored conversation turns.
+ *
+ * A realtime voice model reports a call as separate, loosely ordered events:
+ * final user transcripts, assistant text deltas, tool calls, and the added
+ * assistant message that closes a reply. This recorder groups them into
+ * exchanges and hands each finished exchange to a `write` function (normally
+ * `NeuroLink.recordConversationTurn`).
+ *
+ * It knows nothing about LiveKit, so it can be driven by a synthetic event
+ * stream in tests. The rules it applies:
+ *
+ *  - An exchange is the user's final transcript(s) plus the assistant message
+ *    that answers them. The assistant message's id is the exchange's `turnId`,
+ *    which makes a retried write idempotent.
+ *  - Settle rule. A user's final transcript can arrive after the assistant has
+ *    started, or even finished, replying. A closed reply with no user text is
+ *    held for `settleMs`; a transcript arriving in that window completes it.
+ *    A reply still without user text after the window (the opening greeting,
+ *    typically) is dropped, because a stored turn needs both sides.
+ *  - A new user transcript arriving while the previous reply is still being
+ *    spoken, and after that exchange already had user text, closes the
+ *    previous exchange as interrupted (barge-in) and starts a new one.
+ *  - `flush` is for shutdown. It closes the open exchange with what was said
+ *    so far, marks it interrupted and call-ended, and appends a short note so
+ *    a later reader knows the reply is incomplete. A user turn with no reply
+ *    gets the note alone. The flush waits for pending writes only up to a
+ *    budget, because the worker process is reaped about a second after the
+ *    shutdown callbacks start.
+ *  - Writes run strictly one at a time, in order. A failed write is retried;
+ *    after the last retry it is reported through `onPermanentFailure` and the
+ *    recorder carries on — a storage problem never stops a call.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import type { VoiceTurnRecorder, VoiceTurnRecorderOptions } from "../../types/index.js";
+/** Create a recorder for one call. */
+export declare function createVoiceTurnRecorder(options: VoiceTurnRecorderOptions): VoiceTurnRecorder;