@juspay/neurolink 13.7.0 → 13.8.1

package/dist/types/server.d.ts

@@ -9,7 +9,7 @@ import type { JsonObject, JsonValue } from "./common.js";
 import type { ExternalMCPServerStatus } from "./externalMcp.js";
 import type { ProxyRuntimeConfigProvider } from "./proxy.js";
 import type { TTSAudioFormat } from "./tts.js";
-import type { STTCorrectionOptions, STTDictionaryEntry, STTFallbackOptions, STTDecisionRecord, STTEngineInfo, STTStreamingOptions, TranscribeTimings } from "./transcribe.js";
+import type { STTCorrectionOptions, STTDecisionRecord, STTDictionaryEntry, STTEngineInfo, STTFallbackOptions, STTOutputOptions, STTOutputRendering, STTRoutingOptions, STTStreamingOptions, TranscribeTimings } from "./transcribe.js";
 /**
  * Server adapter configuration
  */
@@ -124,10 +124,18 @@ export type RequiredRateLimitConfig = {
 export type BodyParserConfig = {
     /** Enable body parsing (default: true) */
     enabled?: boolean;
-    /** Maximum body size (default: "10mb") */
+    /** Maximum body size for parsed bodies (default: "10mb") */
     maxSize?: string;
     /** JSON body limit (default: "10mb") */
     jsonLimit?: string;
+    /**
+     * Maximum size of a raw upload: `multipart/form-data`, `audio/*` and
+     * `application/octet-stream` bodies, which reach a route as bytes
+     * (the transcription routes). Default: `maxSize` when set, else "64mb";
+     * the transcription routes apply their own `maxAudioBytes` (50 MB)
+     * beneath it.
+     */
+    rawMaxSize?: string;
     /** Enable URL-encoded body parsing */
     urlEncoded?: boolean;
 };
@@ -138,6 +146,7 @@ export type RequiredBodyParserConfig = {
     enabled: boolean;
     maxSize: string;
     jsonLimit: string;
+    rawMaxSize: string;
     urlEncoded: boolean;
 };
 /**
@@ -1170,6 +1179,37 @@ export type CreateRoutesOptions = {
     agent?: ServerAgentRouteOptions;
 };
 /** Options for `createTranscribeRoutes()`. */
+/**
+ * What `GET /agent/transcribe/providers` reports: the engines this server
+ * can transcribe with, which one a request without a provider gets, and the
+ * models behind the correction layer. Never a key or a URL with a secret.
+ */
+export type ServerTranscribeCapabilities = {
+    /** The provider a request without one goes to. */
+    default: string;
+    providers: Array<{
+        name: string;
+        /** `builtin` ships with NeuroLink, `endpoint` comes from `NEUROLINK_STT_ENDPOINTS`, `registered` was added at runtime. */
+        kind: "builtin" | "endpoint" | "registered";
+        model?: string;
+        /** The endpoint can answer a language verdict without transcribing (batch probe). */
+        languageProbe?: boolean;
+        /** The endpoint takes a language prior between utterances. */
+        languagePrior?: boolean;
+    }>;
+    correction: {
+        /** The decision model that judges dictionary candidates; `null` when none is configured (the dictionary is then applied as is). */
+        decision: {
+            provider: string;
+            model: string;
+        } | null;
+        /** The text model of the rewrite; `provider` is `null` when the instance default applies. */
+        rewrite: {
+            provider: string | null;
+            model: string | null;
+        };
+    };
+};
 export type ServerTranscribeRouteOptions = {
     /**
      * Directories a request's `audioPath` may point into. Empty or omitted (the
@@ -1185,6 +1225,165 @@ export type ServerTranscribeRouteOptions = {
      * `http://host:port/v1` works unchanged.
      */
     openaiBasePath?: string;
+    /**
+     * The STT evaluation collector, `POST <openaiBasePath>/v1/evals/stt`.
+     * On by default; `false` removes its routes.
+     */
+    evals?: ServerSTTEvalOptions | false;
+};
+/** Where and how much the STT evaluation collector keeps. */
+export type ServerSTTEvalOptions = {
+    /**
+     * Directory for one JSON-lines file per call, replayed on start. Default
+     * `NEUROLINK_STT_EVAL_DIR`; without either, calls live in memory only.
+     */
+    dir?: string;
+    /** Calls held in memory; the least recently updated are evicted (still on disk). Default 1000. */
+    maxCalls?: number;
+    /** Segments kept per call; later new ids are counted as `ignored`. Default 50,000. */
+    maxSegmentsPerCall?: number;
+};
+/** One finalized transcript segment, from the STT people saw or the one run alongside it. */
+export type ServerSTTEvalSegment = {
+    /** Stable per segment: a resend with the same id replaces it. */
+    id: string;
+    /** `primary` = shown to users, `shadow` = run for evaluation only. */
+    source: "primary" | "shadow";
+    model: string;
+    participant_identity: string;
+    participant_name?: string;
+    text: string;
+    language?: string | null;
+    /** Epoch seconds, when the STT reports timings. */
+    start_time?: number | null;
+    end_time?: number | null;
+    /** Epoch seconds the sender received the final. */
+    received_at: number;
+};
+/** Body of `POST /v1/evals/stt`: one batch of a call's segments. */
+export type ServerSTTEvalBatchRequest = {
+    call_id: string;
+    room?: string;
+    /** 1, 2, … per call. Numbers may skip when a send failed and was folded into the next. */
+    batch: number;
+    /** True on the batch sent after the last participant left. */
+    final?: boolean;
+    models: {
+        primary: string;
+        shadow: string;
+    };
+    segments: ServerSTTEvalSegment[];
+};
+/** Answer to `POST /v1/evals/stt`. */
+export type ServerSTTEvalBatchResponse = {
+    call_id: string;
+    batch: number;
+    final: boolean;
+    /** New segment ids. */
+    inserted: number;
+    /** Ids already held (a resend), replaced. */
+    updated: number;
+    /** New ids refused because the call is at `maxSegmentsPerCall`. */
+    ignored: number;
+    segments: {
+        primary: number;
+        shadow: number;
+    };
+};
+/** One call in `GET /v1/evals/stt`. */
+export type ServerSTTEvalCallSummary = {
+    call_id: string;
+    room: string | null;
+    models: {
+        primary: string;
+        shadow: string;
+    };
+    final: boolean;
+    batches: number;
+    segments: {
+        primary: number;
+        shadow: number;
+    };
+    speakers: number;
+    first_received_at: number | null;
+    last_received_at: number | null;
+    /** Epoch ms this server last accepted a batch. */
+    updated_at: number;
+};
+/** One STT's side of a speaker's comparison. */
+export type ServerSTTEvalSide = {
+    model: string;
+    segments: number;
+    words: number;
+    /** Languages the STT reported, most frequent first. */
+    languages: string[];
+    /** Mean seconds from the end of speech to the final, when the STT reports timings. */
+    mean_latency_s: number | null;
+};
+/**
+ * Agreement between the two STTs for one speaker. Neither side is ground
+ * truth: `disagreement` is the word edit distance between them divided by the
+ * primary's word count, so 0 means both wrote the same words.
+ */
+export type ServerSTTEvalComparison = {
+    participant_identity: string;
+    participant_name: string;
+    primary: ServerSTTEvalSide;
+    shadow: ServerSTTEvalSide;
+    /** Word edit distance / primary words, on normalized text. Null when the primary wrote nothing. */
+    disagreement: number | null;
+    /** The same on phonetic word keys, so one STT writing Devanagari and the other Latin is not counted as a difference. */
+    phonetic_disagreement: number | null;
+    /** `full` aligns the whole speaker; `windowed` aligns 5-minute windows of a long speaker. */
+    method: "full" | "windowed";
+};
+/**
+ * Where the STT evaluation collector exports to Langfuse. Read only from the
+ * server's environment (`NEUROLINK_STT_EVAL_LANGFUSE_*`): neither a request
+ * nor a route option can redirect evaluation data.
+ */
+export type ServerSTTEvalLangfuseConfig = {
+    publicKey: string;
+    secretKey: string;
+    /** Default `https://cloud.langfuse.com`. */
+    baseUrl: string;
+    environment?: string;
+};
+/** One event of Langfuse's public ingestion API (`POST /api/public/ingestion`). */
+export type ServerSTTEvalLangfuseEvent = {
+    id: string;
+    timestamp: string;
+    type: "trace-create" | "event-create" | "score-create";
+    body: Record<string, unknown>;
+};
+/** A call as the evaluation store holds it in memory. */
+export type ServerSTTEvalCallRecord = {
+    callId: string;
+    room: string | null;
+    models: {
+        primary: string;
+        shadow: string;
+    };
+    final: boolean;
+    batches: Set<number>;
+    segments: Map<string, ServerSTTEvalSegment>;
+    updatedAt: number;
+};
+/** Word edit distance between two readings, and the reference's word count. */
+export type ServerSTTEvalAlignment = {
+    distance: number;
+    refWords: number;
+};
+/** `GET /v1/evals/stt/:callId`. */
+export type ServerSTTEvalReport = ServerSTTEvalCallSummary & {
+    /** Over every speaker: summed edit distances / summed primary words. */
+    overall: {
+        disagreement: number | null;
+        phonetic_disagreement: number | null;
+    };
+    speakers_compared: ServerSTTEvalComparison[];
+    /** Present with `?segments=true`, ordered by time. */
+    segment_list?: ServerSTTEvalSegment[];
 };
 /**
  * Body of `POST <basePath>/agent/transcribe`. Exactly one of `audio`,
@@ -1206,6 +1405,13 @@ export type ServerTranscribeRequest = {
     dictionary?: STTDictionaryEntry[];
     correction?: STTCorrectionOptions;
     fallback?: STTFallbackOptions;
+    routing?: STTRoutingOptions;
+    /** `"utterances"` runs the recording through the live pipeline (see `TranscribeOptions.chunking`). */
+    chunking?: "engine" | "utterances";
+    /** Renderings in other languages and scripts (see `TranscribeOptions.output`). */
+    output?: Omit<STTOutputOptions, "languages"> & {
+        languages: string[];
+    };
     diarization?: boolean;
     wordTimestamps?: boolean;
     timeoutMs?: number;
@@ -1251,6 +1457,8 @@ export type ServerOpenAITranscriptionResponse = {
     language_detected?: boolean;
     steps: string[];
     timings: TranscribeTimings;
+    /** NeuroLink extension: one rendering per requested output language. */
+    outputs?: Record<string, STTOutputRendering>;
 };
 /** Options for `attachTranscribeWebSocket()`. */
 export type ServerTranscribeWebSocketOptions = {
@@ -1273,6 +1481,100 @@ export type ServerTranscribeWebSocketOptions = {
     allowedOrigins?: readonly string[];
     /** Largest single frame, in bytes. Default 1 MiB. */
     maxPayload?: number;
+    /**
+     * Listener path of transcription sessions: a socket here reads one
+     * session's speakers in its own languages. Default `<path>/subscribe`;
+     * `false` turns sessions off (a `?session=` on the stream path is then
+     * ignored).
+     */
+    subscribePath?: string | false;
+    /**
+     * OpenAI Realtime transcription path (`?intent=transcription`): the
+     * protocol of OpenAI's `/v1/realtime`, so any OpenAI-Realtime client — the
+     * LiveKit `openai.STT(use_realtime=True)` plugin, Pipecat, the OpenAI SDK —
+     * streams to NeuroLink with no adapter. Default `/v1/realtime`; `false`
+     * turns it off.
+     */
+    realtimePath?: string | false;
+    /**
+     * Profiles a realtime client selects with its `model` name. Merged over
+     * `NEUROLINK_STT_REALTIME_PROFILES` (a JSON object of the same shape);
+     * these win on a name in both.
+     */
+    realtimeProfiles?: Readonly<Record<string, ServerSTTRealtimeProfile>>;
+};
+/**
+ * What an OpenAI-Realtime client gets when it names this profile as its
+ * `model`. A realtime client can send only a model, a language, a prompt and
+ * turn-detection numbers, so everything NeuroLink-specific (engine, fallback,
+ * dictionary, correction, output languages) is the operator's, here.
+ *
+ * A `model` that is no profile but a configured STT provider uses that
+ * provider with the defaults; any other name (OpenAI's own
+ * `gpt-4o-mini-transcribe`, say) uses the profile named `default`, or the
+ * server's defaults when there is none.
+ */
+export type ServerSTTRealtimeProfile = Omit<ServerTranscribeStreamConfig, "mode"> & {
+    /**
+     * What `…input_audio_transcription.completed` carries: the corrected text
+     * (`"corrected"`, default; it waits up to `correctionWaitMs` for the
+     * correction, then sends the raw final) or the raw final at once.
+     */
+    completed?: "corrected" | "final";
+    /** Longest wait for the correction before `completed` goes out with the final. Default 3000. */
+    correctionWaitMs?: number;
+    /**
+     * Whether the client's `language` pins the transcription. Default `true`.
+     * Note the LiveKit plugin sends `"en"` unless `detect_language=True`.
+     */
+    clientLanguage?: boolean;
+    /**
+     * Whether the client's `turn_detection.silence_duration_ms` sets the
+     * utterance's closing pause. Default `false`: NeuroLink's endpointer keeps
+     * its own (750 ms), since a client default such as LiveKit's 350 ms cuts
+     * conversational speech mid-sentence.
+     */
+    clientTurnDetection?: boolean;
+};
+/** The parts of a `ws` socket the realtime handler uses. */
+export type ServerRealtimeSocket = {
+    readonly readyState: number;
+    send(data: string): void;
+    close(code?: number, reason?: string): void;
+    on(event: "message", listener: (data: Buffer | ArrayBuffer | Buffer[], isBinary: boolean) => void): unknown;
+    on(event: "close" | "error", listener: () => void): unknown;
+};
+/** What a realtime client configured, read from either session dialect. */
+export type ServerRealtimeSessionConfig = {
+    dialect: "ga" | "beta";
+    rate: number;
+    model?: string;
+    language?: string;
+    prompt?: string;
+    silenceMs?: number;
+};
+/** One utterance on the realtime wire: a conversation item. */
+export type ServerRealtimeItem = {
+    utterance: number;
+    itemId: string;
+    /** `speech_started` went out. */
+    started: boolean;
+    /** The committed prefix already sent as deltas. */
+    sent: string;
+    /** Server events held until every earlier item has completed. */
+    pending: Array<{
+        type: string;
+        body: Record<string, unknown>;
+    }>;
+    finalText?: string;
+    engine?: string;
+    language?: string;
+    languageDetected?: boolean;
+    corrected: boolean;
+    /** The transcript `completed` will carry, once known. */
+    ready?: string;
+    done: boolean;
+    timer?: ReturnType<typeof setTimeout>;
 };
 /** Data stream finish event. */
 export type FinishEvent = DataStreamEvent & {

package/dist/types/stt.d.ts

@@ -45,6 +45,13 @@ export type STTOptions = {
      * `vocabulary` terms are appended by the handlers that accept a prompt.
      */
     prompt?: string;
+    /**
+     * The language a previous utterance was confidently detected as. Engines
+     * that take a prior (the IndicConformer wrapper, through its
+     * `priorLanguageForm`) keep it unless the audio clearly says otherwise;
+     * every other engine ignores it. Only sent when no `language` is pinned.
+     */
+    languagePrior?: string;
     /** Per-request timeout in milliseconds, for handlers that honour one. */
     timeoutMs?: number;
     /** Minimum confidence threshold */
@@ -97,6 +104,12 @@ export type STTResult = {
         provider?: string;
         /** Model used */
         model?: string;
+        /** Set by the output guards: what the engine produced was not speech. */
+        hallucination?: "prompt-echo";
+        /** Share of the words a collapsed decoding loop had taken (0 when none). */
+        loopShare?: number;
+        /** Set by language routing: the provider whose text this is, when not the pass's own engine. */
+        routedProvider?: string;
         /** What `confidence` was derived from, for providers that must choose. */
         confidenceSource?: STTConfidenceSource;
         /** Additional provider-specific metadata */
@@ -175,6 +188,13 @@ export type STTHandler = {
      * what to do rather than "set the required API keys".
      */
     describeConfiguration?(): string;
+    /**
+     * The model name that runs this engine in language-identification-only
+     * mode (no transcript, just the language verdict and scores, in a fraction
+     * of the time), when the server has one. The batch path probes with it
+     * before deciding which engine transcribes.
+     */
+    languageProbeModel?(): string | undefined;
     maxAudioDuration?: number;
     supportsStreaming?: boolean;
 };
@@ -460,6 +480,17 @@ export type WhisperTranscriptionSegment = {
      * segment with a speaker; the handler surfaces it as `TranscriptionSegment.speaker`.
      */
     speaker?: string;
+    /**
+     * Extension, same meaning as the response-level fields: a gateway that
+     * normalises the top level of `verbose_json` to OpenAI's schema (LiteLLM
+     * does) keeps unknown keys inside segments, so a server may repeat its
+     * language-detection verdict here and it survives the hop.
+     */
+    language_detected?: boolean;
+    language_scores?: Array<{
+        language: string;
+        score: number;
+    }>;
 };
 export type WhisperVerboseResponse = {
     task: string;
@@ -692,14 +723,18 @@ export type GeminiResponse = {
 };
 /** The three files the built-in Whistle engine needs on disk. */
 export type WhistleAssetName = "needle.js" | "needle.wasm" | "whistle.cact";
-/** One pinned Whistle asset: where it comes from and what it must hash to. */
-export type WhistleAsset = {
-    name: WhistleAssetName;
+/** One pinned model file NeuroLink ships or fetches on first use: where it comes from and what it must hash to. */
+export type LocalModelAsset = {
+    name: string;
     url: string;
     sha256: string;
     /** Exact size in bytes, used for the download notice and a cheap pre-check. */
     bytes: number;
 };
+/** One pinned Whistle asset. */
+export type WhistleAsset = LocalModelAsset & {
+    name: WhistleAssetName;
+};
 /** Absolute paths of the three Whistle files once they are present and verified. */
 export type WhistleAssetPaths = Record<WhistleAssetName, string>;
 /** One word as the Whistle engine reports it (seconds from the start of the audio). */