@juspay/neurolink 12.48.2 → 12.49.0

package/dist/types/index.js

@@ -81,6 +81,7 @@ export * from "./tools.js";
 export * from "./vectorStoreChroma.js";
 export * from "./vectorStorePinecone.js";
 export * from "./voice.js";
+export * from "./transcribe.js";
 export * from "./utilities.js";
 export * from "./workflow.js";
 // Processor base types are re-exported via ./processor.js

package/dist/types/providers.d.ts

@@ -5,6 +5,7 @@ import type { UnknownRecord, JsonValue, StreamingCapability } from "./common.js"
 import type { NeuroLink } from "../neurolink.js";
 import { AIProviderName, AnthropicModels, BedrockModels, DeepSeekModels, GoogleAIModels, LlamaCppModels, LMStudioModels, NvidiaNimModels, OpenAIModels, VertexModels } from "../constants/enums.js";
 import type { ValidationSchema } from "./aliases.js";
+import type { STTCredentials } from "./transcribe.js";
 import type { DecisionLimits, DecisionRequest, DecisionResult } from "./decision.js";
 import type { FileToolRootPolicy } from "./fileToolRoots.js";
 import type { EnhancedGenerateResult, TextGenerationOptions } from "./generate.js";
@@ -105,6 +106,11 @@ export type NeurolinkCredentials = {
         apiKey?: string;
         baseURL?: string;
     };
+    /**
+     * Speech-to-text handlers (`transcribe()`, and `generate`/`stream` with
+     * `stt`). One slice per STT provider; see `STTCredentials`.
+     */
+    stt?: STTCredentials;
     /**
      * Anthropic. `baseURL` points the official SDK client at a gateway or proxy
      * instead of `api.anthropic.com` (with or without a trailing `/v1`); it
@@ -2293,7 +2299,7 @@ export type ProviderRegistration = {
  * calibrated judgements and no text at all — see src/lib/types/decision.ts.
  * They are peers: a provider may serve any subset.
  */
-export type InferenceKind = "generate" | "stream" | "decide";
+export type InferenceKind = "generate" | "stream" | "decide" | "transcribe";
 /** What a descriptor means when it declares no `inferenceKinds`. */
 export declare const DEFAULT_INFERENCE_KINDS: readonly InferenceKind[];
 /**

package/dist/types/server.d.ts

@@ -8,6 +8,8 @@ import type { NeuroLink } from "../neurolink.js";
 import type { JsonObject, JsonValue } from "./common.js";
 import type { ExternalMCPServerStatus } from "./externalMcp.js";
 import type { ProxyRuntimeConfigProvider } from "./proxy.js";
+import type { TTSAudioFormat } from "./tts.js";
+import type { STTCorrectionOptions, STTDictionaryEntry, STTFallbackOptions, STTDecisionRecord, STTEngineInfo, STTStreamingOptions, TranscribeTimings } from "./transcribe.js";
 /**
  * Server adapter configuration
  */
@@ -63,7 +65,10 @@ export type RequiredServerAdapterConfig = {
 export type CORSConfig = {
     /** Enable CORS (default: true) */
     enabled?: boolean;
-    /** Allowed origins (default: ["*"]) */
+    /**
+     * Allowed browser origins. Default `[]`: no cross-origin page may call the
+     * server until an origin is listed; `["*"]` opts in to any origin.
+     */
     origins?: string[];
     /** Allowed HTTP methods */
     methods?: string[];
@@ -1095,6 +1100,117 @@ export type CreateRoutesOptions = {
      * docs/features/codex-proxy-support.md.
      */
     runtimeConfigProvider?: ProxyRuntimeConfigProvider;
+    /**
+     * Options for the transcription routes (`POST <basePath>/agent/transcribe`
+     * and the OpenAI-compatible `POST /v1/audio/transcriptions`), which are
+     * always mounted.
+     */
+    transcribe?: ServerTranscribeRouteOptions;
+};
+/** Options for `createTranscribeRoutes()`. */
+export type ServerTranscribeRouteOptions = {
+    /**
+     * Directories a request's `audioPath` may point into. Empty or omitted (the
+     * default) refuses every `audioPath`: a server must not read its own disk
+     * on a caller's say-so unless the operator names where.
+     */
+    allowedAudioRoots?: string[];
+    /** Largest audio accepted, decoded, in bytes. Default 50 MB. */
+    maxAudioBytes?: number;
+    /**
+     * Prefix of the OpenAI-compatible route. Default `""`, so the route is
+     * `/v1/audio/transcriptions` and an OpenAI client pointed at
+     * `http://host:port/v1` works unchanged.
+     */
+    openaiBasePath?: string;
+};
+/**
+ * Body of `POST <basePath>/agent/transcribe`. Exactly one of `audio`,
+ * `audioUrl` or `audioPath`. There is no `credentials` field: a caller cannot
+ * swap in its own key or point an engine at another host.
+ */
+export type ServerTranscribeRequest = {
+    /** Base64 audio, or a `data:` URL. */
+    audio?: string;
+    /** `http(s)` URL the server downloads (SSRF-guarded, size-capped). */
+    audioUrl?: string;
+    /** Server-local path; refused unless it lies under `allowedAudioRoots`. */
+    audioPath?: string;
+    format?: TTSAudioFormat;
+    provider?: string;
+    model?: string;
+    language?: string;
+    prompt?: string;
+    dictionary?: STTDictionaryEntry[];
+    correction?: STTCorrectionOptions;
+    fallback?: STTFallbackOptions;
+    diarization?: boolean;
+    wordTimestamps?: boolean;
+    timeoutMs?: number;
+};
+/** The transcribe WebSocket's first text frame: the JSON route's fields without the audio. */
+export type ServerTranscribeStreamConfig = Omit<ServerTranscribeRequest, "audio" | "audioUrl" | "audioPath" | "format"> & {
+    streaming?: STTStreamingOptions;
+};
+/** One segment of the OpenAI `verbose_json` transcription shape. */
+export type ServerOpenAITranscriptionSegment = {
+    id: number;
+    start: number;
+    end: number;
+    text: string;
+    /** NeuroLink extension: speaker label when diarization ran. */
+    speaker?: string;
+};
+/** One word of the OpenAI `verbose_json` transcription shape. */
+export type ServerOpenAITranscriptionWord = {
+    word: string;
+    start: number;
+    end: number;
+    /** NeuroLink extension: speaker label when diarization ran. */
+    speaker?: string;
+};
+/**
+ * `POST /v1/audio/transcriptions` with `response_format=verbose_json`:
+ * OpenAI's shape, plus NeuroLink's extras in snake_case.
+ */
+export type ServerOpenAITranscriptionResponse = {
+    task: "transcribe";
+    text: string;
+    language?: string;
+    duration?: number;
+    segments?: ServerOpenAITranscriptionSegment[];
+    words?: ServerOpenAITranscriptionWord[];
+    /** Engine text before correction. */
+    raw: string;
+    /** Present when the correction changed the text. */
+    corrected?: string;
+    decisions?: STTDecisionRecord[];
+    engine: STTEngineInfo;
+    language_detected?: boolean;
+    steps: string[];
+    timings: TranscribeTimings;
+};
+/** Options for `attachTranscribeWebSocket()`. */
+export type ServerTranscribeWebSocketOptions = {
+    /** Upgrade path. Default `/v1/audio/transcriptions/stream`. */
+    path?: string;
+    /**
+     * Token(s) required on the upgrade (`Authorization: Bearer` header or
+     * `?token=`). `neurolink serve` passes the same keys as its HTTP routes
+     * (`NEUROLINK_SERVER_API_KEY`, comma-separated), so one setting guards both.
+     */
+    authToken?: string | readonly string[];
+    /**
+     * Browser origins allowed to upgrade. A browser always sends `Origin` on a
+     * WebSocket upgrade, and an upgrade whose `Origin` is not listed is refused
+     * with 403; a request without the header (a non-browser client) is let
+     * through, as CORS does not apply to it. Omitted, or containing `"*"`, means
+     * any origin. `neurolink serve` passes its `cors.origins` here so one
+     * allow-list covers the HTTP routes and the stream.
+     */
+    allowedOrigins?: readonly string[];
+    /** Largest single frame, in bytes. Default 1 MiB. */
+    maxPayload?: number;
 };
 /** Data stream finish event. */
 export type FinishEvent = DataStreamEvent & {

package/dist/types/stt.d.ts

@@ -39,6 +39,14 @@ export type STTOptions = {
     model?: string;
     /** Custom vocabulary/phrases */
     vocabulary?: string[];
+    /**
+     * Context prompt for engines that bias on one (Whisper and other
+     * OpenAI-compatible servers). Engines without a prompt ignore it;
+     * `vocabulary` terms are appended by the handlers that accept a prompt.
+     */
+    prompt?: string;
+    /** Per-request timeout in milliseconds, for handlers that honour one. */
+    timeoutMs?: number;
     /** Minimum confidence threshold */
     confidenceThreshold?: number;
     /**
@@ -160,6 +168,13 @@ export type STTHandler = {
     getSupportedLanguages?(): Promise<STTLanguage[]>;
     getSupportedFormats(): TTSAudioFormat[];
     isConfigured(): boolean;
+    /**
+     * What would make `isConfigured()` true, in the handler's own words — the
+     * env var or key for a hosted engine, the files and their download URLs for
+     * a local one. Appended to the "not configured" error so the caller is told
+     * what to do rather than "set the required API keys".
+     */
+    describeConfiguration?(): string;
     maxAudioDuration?: number;
     supportsStreaming?: boolean;
 };
@@ -262,7 +277,12 @@ export type GoogleSTTOptions = STTOptions & {
     useEnhanced?: boolean;
     keywords?: string[];
 };
-export type WhisperModel = "whisper-1";
+/**
+ * `whisper-1` is OpenAI's own model; any other string is passed through as-is,
+ * which is what a self-hosted OpenAI-compatible transcription server (vLLM,
+ * LiteLLM, a diarization sidecar) expects — e.g. `"qwen3-asr"`.
+ */
+export type WhisperModel = "whisper-1" | (string & NonNullable<unknown>);
 export type WhisperSTTOptions = STTOptions & {
     model?: WhisperModel;
     responseFormat?: "json" | "text" | "srt" | "verbose_json" | "vtt";
@@ -270,6 +290,12 @@ export type WhisperSTTOptions = STTOptions & {
     prompt?: string;
     /** Translate audio to English instead of transcribing in original language */
     translate?: boolean;
+    /**
+     * Request timeout in milliseconds. Default 30_000, which suits OpenAI's
+     * hosted endpoint; a self-hosted server transcribing a whole meeting needs
+     * minutes, so raise it (or set `OPENAI_STT_TIMEOUT_MS`).
+     */
+    timeoutMs?: number;
 };
 export type AzureWord = {
     Word: string;
@@ -320,6 +346,9 @@ export type DeepgramAlternative = {
 };
 export type DeepgramChannel = {
     alternatives: DeepgramAlternative[];
+    /** Present when the request asked for `detect_language`. */
+    detected_language?: string;
+    language_confidence?: number;
 };
 export type DeepgramUtterance = {
     start: number;
@@ -415,15 +444,22 @@ export type WhisperTranscriptionWord = {
 };
 export type WhisperTranscriptionSegment = {
     id: number;
-    seek: number;
+    /** OpenAI always sends these decoder stats; OpenAI-compatible servers may omit them. */
+    seek?: number;
     start: number;
     end: number;
     text: string;
-    tokens: number[];
-    temperature: number;
-    avg_logprob: number;
-    compression_ratio: number;
-    no_speech_prob: number;
+    tokens?: number[];
+    temperature?: number;
+    avg_logprob?: number;
+    compression_ratio?: number;
+    no_speech_prob?: number;
+    /**
+     * Not part of OpenAI's schema. Diarizing OpenAI-compatible servers (the
+     * NeuroLink diarization sidecar, Deepgram's compat endpoint, …) label each
+     * segment with a speaker; the handler surfaces it as `TranscriptionSegment.speaker`.
+     */
+    speaker?: string;
 };
 export type WhisperVerboseResponse = {
     task: string;
@@ -432,6 +468,20 @@ export type WhisperVerboseResponse = {
     text: string;
     segments?: WhisperTranscriptionSegment[];
     words?: WhisperTranscriptionWord[];
+    /** Extension: distinct speaker labels, in order of first appearance. */
+    speakers?: string[];
+    /**
+     * Extension: a self-hosted server that scores language identification may
+     * say whether it trusts its own detection. Read defensively; never required.
+     */
+    language_detected?: boolean;
+    /** Extension: candidate languages and the server's score for each. */
+    language_scores?: Array<{
+        language: string;
+        score: number;
+    }>;
+    /** Extension: the server's confidence in `language`. */
+    language_confidence?: number;
 };
 export type WhisperSimpleResponse = {
     text: string;
@@ -640,3 +690,106 @@ export type GeminiResponse = {
         ids: string[];
     };
 };
+/** The three files the built-in Whistle engine needs on disk. */
+export type WhistleAssetName = "needle.js" | "needle.wasm" | "whistle.cact";
+/** One pinned Whistle asset: where it comes from and what it must hash to. */
+export type WhistleAsset = {
+    name: WhistleAssetName;
+    url: string;
+    sha256: string;
+    /** Exact size in bytes, used for the download notice and a cheap pre-check. */
+    bytes: number;
+};
+/** Absolute paths of the three Whistle files once they are present and verified. */
+export type WhistleAssetPaths = Record<WhistleAssetName, string>;
+/** One word as the Whistle engine reports it (seconds from the start of the audio). */
+export type WhistleWord = {
+    word: string;
+    start: number;
+    end: number;
+    probability: number;
+};
+/** A finished batch transcription from the Whistle worker, all chunks joined. */
+export type WhistleEngineOutput = {
+    text: string;
+    language: string | null;
+    words: WhistleWord[];
+    /** Time to first token of the first chunk. */
+    ttftMs: number | null;
+    /** Mean decode speed over the chunks, tokens per second. */
+    decodeTps: number | null;
+    /** How many ≤30 s pieces the audio was cut into. */
+    chunks: number;
+};
+/** One step of the engine's native stream: what this chunk committed, and the unconfirmed tail. */
+export type WhistleStreamStep = {
+    text: string;
+    words: WhistleWord[];
+    pending: string;
+    language: string | null;
+    /** Seconds of audio received so far. */
+    received: number;
+    passMs: number;
+};
+/** Messages the handler sends to the Whistle worker thread. */
+export type WhistleWorkerRequest = {
+    id: number;
+    op: "transcribe";
+    pcm: Float32Array;
+    language: string | null;
+    keywords: string | null;
+    timestamps: boolean;
+} | {
+    id: number;
+    op: "streamProcess";
+    pcm: Float32Array;
+    language: string | null;
+    keywords: string | null;
+} | {
+    id: number;
+    op: "streamStop";
+};
+/** Messages the Whistle worker thread sends back. */
+export type WhistleWorkerResponse = {
+    kind: "ready";
+    loadMs: number;
+} | {
+    kind: "bootError";
+    error: string;
+} | {
+    kind: "transcribed";
+    id: number;
+    result: WhistleEngineOutput;
+} | {
+    kind: "streamStep";
+    id: number;
+    result: WhistleStreamStep;
+} | {
+    kind: "failed";
+    id: number;
+    error: string;
+};
+/** `workerData` for the Whistle worker thread. */
+export type WhistleWorkerData = {
+    paths: WhistleAssetPaths;
+};
+/** The subset of the Emscripten "needle" module the Whistle worker calls. */
+export type WhistleNeedleModule = {
+    HEAPU8: Uint8Array;
+    _malloc(size: number): number;
+    _free(ptr: number): void;
+    _needle_load(ptr: number, length: bigint): number;
+    _needle_last_error(): number;
+    _needle_transcribe(pcm: number, samples: number, language: number, keywords: number, timestamps: number, out: number, outCapacity: number): number;
+    _needle_stream_transcribe_process(pcm: number, samples: number, language: number, keywords: number, out: number, outCapacity: number): number;
+    _needle_stream_transcribe_stop(out: number, outCapacity: number): number;
+    UTF8ToString(ptr: number): string;
+};
+/** `createNeedle` — the factory `needle.js` exports. */
+export type WhistleNeedleFactory = (moduleArg: {
+    wasmBinary?: Uint8Array;
+    print?: (text: string) => void;
+    printErr?: (text: string) => void;
+}) => Promise<WhistleNeedleModule>;
+/** A worker request before the engine stamps its id (the `Omit` distributes over the union). */
+export type WhistleWorkerOutgoing<T = WhistleWorkerRequest> = T extends unknown ? Omit<T, "id"> : never;