@juspay/neurolink 13.7.0 → 13.8.1

package/dist/server/websocket/transcribeWebSocket.js

@@ -8,73 +8,49 @@
  * 2. Then BINARY frames of PCM16LE mono audio at `streaming.sampleRate`
  *    (default 16000).
  * 3. A TEXT frame `{"type":"end"}` (or closing the socket) ends the audio; the
- *    server flushes the last utterance.
+ *    server flushes the last utterance. `{"type":"output","languages":[…]}`
+ *    changes the output languages from the next utterance on; any other text
+ *    frame ends the audio, as before.
  *
  * The server sends one TEXT frame per `TranscribeStreamEvent` (JSON), then
  * closes with 1000. A bad config is answered with an `error` event and close
  * code 1008.
  *
+ * Sessions (one call, several speakers, listeners in their own languages):
+ * a speaker connects to the stream path with `?session=<id>&speaker=<name>`
+ * and uses the protocol above; a listener connects to the subscribe path
+ * (default `<path>/subscribe`) with `?session=<id>` and sends
+ * `{"type":"subscribe","languages":["es"],"speakers":[…]}` — again at any time
+ * to change them. Listeners receive every event of the speakers they follow,
+ * tagged with `speaker`, and only the `output` events of their languages; a
+ * session renders each utterance once into the union of its listeners'
+ * languages. A session lives while a speaker or a listener is connected.
+ *
+ * OpenAI Realtime transcription (default path `/v1/realtime`,
+ * `?intent=transcription`): the same pipeline behind OpenAI's protocol, for
+ * clients that already speak it (see realtimeTranscription.ts).
+ *
  * The route adapters do not handle upgrades, so this attaches to the Node
  * `http.Server` the caller owns, next to whatever else is listening there; it
  * answers only upgrades on its own path.
  *
  * @module server/websocket/transcribeWebSocket
  */
+import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import { logger } from "../../utils/logger.js";
 import { timingSafeEqualString } from "../voice/tokenCompare.js";
-import { TranscribeStreamConfigSchema } from "../utils/validation.js";
+import { TranscribeOutputControlSchema, TranscribeStreamConfigSchema, TranscribeSubscribeSchema, } from "../utils/validation.js";
+import { fileModeStreamOptions } from "../../voice/transcribe.js";
+import { TranscriptionSession } from "../../voice/session/transcriptionSession.js";
+import { FrameQueue } from "./frameQueue.js";
+import { resolveRealtimeProfiles, serveRealtimeTranscription, } from "./realtimeTranscription.js";
 const DEFAULT_PATH = "/v1/audio/transcriptions/stream";
+const DEFAULT_REALTIME_PATH = "/v1/realtime";
 /** Audio waiting for the engine beyond this (about 8.5 min at 16 kHz) closes the socket. */
 const MAX_QUEUED_BYTES = 16 * 1024 * 1024;
-/** Push-driven async iterable: socket frames in, `transcribeStream()` pulls. */
-class FrameQueue {
-    frames = [];
-    waiting;
-    ended = false;
-    queuedBytes = 0;
-    push(frame) {
-        if (this.ended) {
-            return;
-        }
-        if (this.waiting) {
-            const resolve = this.waiting;
-            this.waiting = undefined;
-            resolve({ value: frame, done: false });
-            return;
-        }
-        this.frames.push(frame);
-        this.queuedBytes += frame.length;
-    }
-    end() {
-        this.ended = true;
-        if (this.waiting) {
-            const resolve = this.waiting;
-            this.waiting = undefined;
-            resolve({ value: undefined, done: true });
-        }
-    }
-    [Symbol.asyncIterator]() {
-        return {
-            next: () => {
-                const frame = this.frames.shift();
-                if (frame) {
-                    this.queuedBytes -= frame.length;
-                    return Promise.resolve({ value: frame, done: false });
-                }
-                if (this.ended) {
-                    return Promise.resolve({ value: undefined, done: true });
-                }
-                return new Promise((resolve) => {
-                    this.waiting = resolve;
-                });
-            },
-            return: () => {
-                this.end();
-                return Promise.resolve({ value: undefined, done: true });
-            },
-        };
-    }
-}
+/** In file mode the whole recording may be queued at once: 512 MB is about 4.5 hours of 16 kHz PCM. */
+const MAX_QUEUED_BYTES_FILE = 512 * 1024 * 1024;
 /**
  * `true` when the upgrade's `Origin` is on the allow-list, the list allows
  * any origin, or the request carries no `Origin` (not a browser).
@@ -90,7 +66,7 @@ function isOriginAllowed(req, allowed) {
     const wanted = origin.trim().toLowerCase();
     return allowed.some((o) => o.trim().toLowerCase() === wanted);
 }
-function isAuthorized(req, token) {
+function isAuthorized(req, token, allowSubprotocolKey = false) {
     const tokens = (typeof token === "string" ? [token] : (token ?? [])).filter((t) => t.length > 0);
     if (tokens.length === 0) {
         return true;
@@ -108,7 +84,16 @@ function isAuthorized(req, token) {
     catch {
         urlToken = undefined;
     }
-    const provided = headerToken ?? urlToken;
+    // OpenAI's browser SDK cannot set headers, so it sends the key as the
+    // subprotocol "openai-insecure-api-key.<key>"; honoured on the realtime path.
+    const protocolToken = allowSubprotocolKey
+        ? String(req.headers["sec-websocket-protocol"] ?? "")
+            .split(",")
+            .map((p) => p.trim())
+            .find((p) => p.startsWith("openai-insecure-api-key."))
+            ?.slice("openai-insecure-api-key.".length)
+        : undefined;
+    const provided = headerToken ?? urlToken ?? protocolToken;
     return (provided !== undefined &&
         tokens.some((t) => timingSafeEqualString(provided, t)));
 }
@@ -120,6 +105,26 @@ function pathOf(req) {
         return "";
     }
 }
+/** A query parameter of the upgrade URL, trimmed and length-capped; `undefined` when absent. */
+function queryParam(req, name) {
+    try {
+        const value = new URL(req.url ?? "/", "http://localhost").searchParams
+            .get(name)
+            ?.trim();
+        return value ? value.slice(0, 100) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function parseJson(frame) {
+    try {
+        return JSON.parse(frame.toString("utf-8"));
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
  * Serve `transcribeStream()` on `path` (default
  * `/v1/audio/transcriptions/stream`) of an existing HTTP server.
@@ -138,12 +143,46 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
     // Loaded on use so `ws` stays off the package's static import graph.
     const { WebSocketServer, WebSocket } = await import("ws");
     const path = options.path ?? DEFAULT_PATH;
+    const subscribePath = options.subscribePath === false
+        ? undefined
+        : (options.subscribePath ?? `${path}/subscribe`);
+    const realtimePath = options.realtimePath === false
+        ? undefined
+        : (options.realtimePath ?? DEFAULT_REALTIME_PATH);
+    const realtimeProfiles = resolveRealtimeProfiles(options.realtimeProfiles);
     const wss = new WebSocketServer({
         noServer: true,
         maxPayload: options.maxPayload ?? 1_048_576,
+        // OpenAI's browser SDK offers "realtime" alongside its key subprotocol.
+        handleProtocols: (protocols) => protocols.has("realtime") ? "realtime" : false,
     });
+    // Sessions by id, alive while a speaker or a listener holds them.
+    const sessions = new Map();
+    const acquire = (id) => {
+        const entry = sessions.get(id) ?? {
+            session: new TranscriptionSession(neurolink, { id }),
+            refs: 0,
+        };
+        entry.refs++;
+        sessions.set(id, entry);
+        return entry.session;
+    };
+    const release = (id) => {
+        const entry = sessions.get(id);
+        if (!entry) {
+            return;
+        }
+        entry.refs--;
+        if (entry.refs <= 0) {
+            entry.session.close();
+            sessions.delete(id);
+        }
+    };
     const onUpgrade = (req, socket, head) => {
-        if (pathOf(req) !== path) {
+        const requested = pathOf(req);
+        if (requested !== path &&
+            requested !== subscribePath &&
+            requested !== realtimePath) {
             return; // Someone else's upgrade.
         }
         if (!isOriginAllowed(req, options.allowedOrigins)) {
@@ -151,7 +190,7 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
             socket.destroy();
             return;
         }
-        if (!isAuthorized(req, options.authToken)) {
+        if (!isAuthorized(req, options.authToken, requested === realtimePath)) {
             socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
             socket.destroy();
             return;
@@ -161,22 +200,164 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
         });
     };
     server.on("upgrade", onUpgrade);
-    wss.on("connection", (client) => {
+    /** A listener: subscribe frames in, the session's events (filtered) out. */
+    const onListener = (client, req) => {
+        const id = queryParam(req, "session");
+        if (!id) {
+            client.send(JSON.stringify({
+                type: "error",
+                message: "A listener needs ?session=<id>",
+                recoverable: false,
+            }));
+            client.close(1008, "session required");
+            return;
+        }
+        const session = acquire(id);
+        let subscription;
+        const deliver = (event) => {
+            if (client.readyState === WebSocket.OPEN) {
+                client.send(JSON.stringify(event));
+            }
+        };
+        client.on("message", (data) => {
+            const frame = Buffer.isBuffer(data)
+                ? data
+                : Array.isArray(data)
+                    ? Buffer.concat(data)
+                    : Buffer.from(data);
+            const parsed = TranscribeSubscribeSchema.safeParse(parseJson(frame));
+            if (!parsed.success) {
+                client.send(JSON.stringify({
+                    type: "error",
+                    message: 'Expected {"type":"subscribe","languages":[…],"speakers":[…]}',
+                    recoverable: true,
+                }));
+                return;
+            }
+            if (subscription) {
+                subscription.update({
+                    ...(parsed.data.languages
+                        ? { languages: parsed.data.languages }
+                        : {}),
+                    ...(parsed.data.speakers ? { speakers: parsed.data.speakers } : {}),
+                });
+                return;
+            }
+            subscription = session.subscribe(parsed.data, deliver);
+        });
+        let released = false;
+        const leave = () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            subscription?.close();
+            release(id);
+        };
+        client.on("close", leave);
+        client.on("error", leave);
+    };
+    wss.on("connection", (client, req) => {
+        if (subscribePath && pathOf(req) === subscribePath) {
+            onListener(client, req);
+            return;
+        }
+        if (realtimePath && pathOf(req) === realtimePath) {
+            serveRealtimeTranscription(client, req, neurolink, realtimeProfiles);
+            return;
+        }
         const queue = new FrameQueue();
         let started = false;
+        let fileMode = false;
+        // Debugging: `NEUROLINK_STT_RECORD_DIR` keeps every session's config,
+        // audio (PCM16LE) and events on disk, so a live session can be replayed
+        // and compared against another engine. Off unless set.
+        const recordRoot = process.env.NEUROLINK_STT_RECORD_DIR?.trim();
+        const recordDir = recordRoot
+            ? join(recordRoot, `${new Date().toISOString().replace(/[:.]/g, "-")}-${Math.random().toString(36).slice(2, 6)}`)
+            : undefined;
+        const record = (name, data, append = true) => {
+            if (!recordDir) {
+                return;
+            }
+            try {
+                mkdirSync(recordDir, { recursive: true });
+                (append ? appendFileSync : writeFileSync)(join(recordDir, name), data);
+            }
+            catch {
+                // Recording is best effort; it never affects the session.
+            }
+        };
+        // Output languages, changeable mid-stream by control frames.
+        let liveLanguages = [];
+        const sessionId = subscribePath ? queryParam(req, "session") : undefined;
+        // No name: the session assigns a unique one (speaker-1, speaker-2, …).
+        const speaker = queryParam(req, "speaker") ?? "";
         const send = (event) => {
+            record("events.jsonl", `${JSON.stringify({ at: Date.now(), ...event })}\n`);
             if (client.readyState !== WebSocket.OPEN) {
                 return false;
             }
             client.send(JSON.stringify(event));
             return true;
         };
+        const events = (config) => {
+            const options = {
+                ...config,
+                audio: queue,
+                output: {
+                    ...config.output,
+                    languages: { current: () => liveLanguages },
+                },
+            };
+            if (!sessionId) {
+                return neurolink.transcribeStream(options);
+            }
+            // A session speaker: events go to the session's listeners and back to
+            // this socket. The session is held until this speaker's audio ends.
+            const session = acquire(sessionId);
+            const buffered = [];
+            let wake;
+            let finished = false;
+            let failure;
+            void session
+                .addSpeaker(speaker, options, (event) => {
+                buffered.push(event);
+                wake?.();
+            })
+                .catch((error) => {
+                failure = error;
+            })
+                .finally(() => {
+                finished = true;
+                release(sessionId);
+                wake?.();
+            });
+            return {
+                [Symbol.asyncIterator]: async function* () {
+                    for (;;) {
+                        const next = buffered.shift();
+                        if (next) {
+                            yield next;
+                            continue;
+                        }
+                        if (finished) {
+                            if (failure) {
+                                throw failure;
+                            }
+                            return;
+                        }
+                        await new Promise((resolve) => {
+                            wake = resolve;
+                        });
+                        wake = undefined;
+                    }
+                },
+            };
+        };
         const run = async (config) => {
             try {
-                for await (const event of neurolink.transcribeStream({
-                    ...config,
-                    audio: queue,
-                })) {
+                for await (const event of events(config)) {
                     if (!send(event)) {
                         break; // The client is gone; stop pulling.
                     }
@@ -187,7 +368,9 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
             }
             catch (error) {
                 const message = error instanceof Error ? error.message : String(error);
-                logger.warn("[TranscribeWebSocket] stream failed", { error: message });
+                logger.warn("[TranscribeWebSocket] stream failed", {
+                    error: message,
+                });
                 send({ type: "error", message, recoverable: false });
                 if (client.readyState === WebSocket.OPEN) {
                     client.close(1011, "transcription failed");
@@ -230,12 +413,18 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
                     return;
                 }
                 started = true;
-                void run(config.data);
+                const { mode, ...rest } = config.data;
+                fileMode = mode === "file";
+                liveLanguages = rest.output?.languages ?? [];
+                record("config.json", JSON.stringify(config.data, null, 1), false);
+                void run(fileMode ? fileModeStreamOptions(rest) : rest);
                 return;
             }
             if (isBinary) {
+                record("audio.pcm", frame);
                 queue.push(frame);
-                if (queue.queuedBytes > MAX_QUEUED_BYTES) {
+                if (queue.queuedBytes >
+                    (fileMode ? MAX_QUEUED_BYTES_FILE : MAX_QUEUED_BYTES)) {
                     send({
                         type: "error",
                         message: "Audio is arriving faster than it can be transcribed",
@@ -246,7 +435,13 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
                 }
                 return;
             }
-            // Any text frame after the config ends the audio ({"type":"end"}).
+            // `output` changes the languages; any other text frame ends the
+            // audio ({"type":"end"}).
+            const control = TranscribeOutputControlSchema.safeParse(parseJson(frame));
+            if (control.success) {
+                liveLanguages = control.data.languages;
+                return;
+            }
             queue.end();
         });
         client.on("close", () => queue.end());
@@ -258,6 +453,10 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
             for (const client of wss.clients) {
                 client.terminate();
             }
+            for (const { session } of sessions.values()) {
+                session.close();
+            }
+            sessions.clear();
             wss.close();
         },
     };

package/dist/services/server/ai/observability/langfuseSetup.js

@@ -1,3 +1,4 @@
+import { DEFAULT_BASE_URL } from "../../../../constants/langfuse.js";
 import { OTEL_LOG_PREFIX } from "../../../../constants/telemetry.js";
 import { LangfuseExporter } from "../../../../observability/exporters/langfuseExporter.js";
 import { logger } from "../../../../utils/logger.js";
@@ -90,11 +91,15 @@ export function createLangfuseExporter(config) {
         logger.warn(`${OTEL_LOG_PREFIX} Langfuse is enabled but publicKey and secretKey are not both set in config or LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY; not exporting to Langfuse`);
         return null;
     }
+    const baseUrl = options.baseUrl || process.env.LANGFUSE_BASE_URL || undefined;
+    if (baseUrl === undefined) {
+        logger.warn(`${OTEL_LOG_PREFIX} Langfuse base URL is not set in observability.langfuse.baseUrl or LANGFUSE_BASE_URL; exporting to ${DEFAULT_BASE_URL}`);
+    }
     const exporter = new LangfuseExporter({
         enabled: true,
         publicKey,
         secretKey,
-        baseUrl: options.baseUrl ?? process.env.LANGFUSE_BASE_URL,
+        baseUrl,
         // The released build labelled Langfuse traces from the option and
         // LANGFUSE_TRACING_ENVIRONMENT / LANGFUSE_RELEASE only (the exporter reads
         // those two itself); the longer chain feeds the OTLP resource alone.

package/dist/types/cli.d.ts

@@ -1142,6 +1142,8 @@ export type ServeCommandArgs = {
     cors?: boolean;
     /** Browser origins allowed by CORS (`--cors-origin`, repeatable or comma-separated). */
     corsOrigin?: string[];
+    /** Largest raw upload (multipart / audio bodies) the server accepts, e.g. "64mb" (`--max-upload`). */
+    maxUpload?: string;
     rateLimit?: number;
     swagger?: boolean;
     config?: string;
@@ -1342,9 +1344,13 @@ export type CliTranscribeArgs = {
     rewrite?: boolean;
     transliterate?: "latin" | "native";
     secondOpinion?: string;
+    /** `--to`: output languages (comma-separated BCP-47 tags). */
+    to?: string | string[];
     fallback?: string;
     context?: string;
     stream?: boolean;
+    /** With `--stream`: which speech detector finds utterances. */
+    vad?: "auto" | "silero" | "energy";
     format?: "text" | "json" | "srt" | "vtt";
     timeout?: number;
     quiet?: boolean;

package/dist/types/exporter.d.ts

@@ -306,7 +306,7 @@ export type SpanAttributeValue = SpanAttributes[string];
 export type OtlpAnyValue = {
     boolValue: boolean;
 } | {
-    intValue: string;
+    intValue: number;
 } | {
     doubleValue: number;
 } | {
@@ -383,6 +383,7 @@ export type OtlpTraceRequest = {
 export type LangfuseDeliveryReport = {
     delivered: number;
     rejected: number;
+    refused: number;
     droppedIds: string[];
     error?: string;
 };
@@ -392,6 +393,7 @@ export type LangfusePostOutcome = {
     rejected: number;
 } | {
     accepted: false;
+    refused?: true;
 };
 /** @description What the Langfuse exporter adds to, masks in and removes from span attributes before sending. */
 export type LangfuseAttributePolicy = Pick<LangfuseExporterConfig, "environment" | "release" | "mask"> & {