@juspay/neurolink 12.48.1 → 12.49.0

package/dist/neurolink.d.ts

@@ -15,7 +15,7 @@ import type { DynamicOptions } from "./types/index.js";
 import { SkillsManager } from "./skills/skillsManager.js";
 import type { CostAnalysisOptions, CostAnalysisResult, ProviderMetricsOptions, ProviderMetricsResult, TeamAnalyticsOptions, TeamAnalyticsResult } from "./types/index.js";
 import { TaskManager } from "./tasks/taskManager.js";
-import type { SessionExport, SessionListItem } from "./types/index.js";
+import type { TranscribeOptions, TranscribeResult, TranscribeStreamEvent, TranscribeStreamOptions, SessionExport, SessionListItem } from "./types/index.js";
 /**
  * Curator P2-4 dedup (concurrency-safe): native providers emit
  * `generation:end` on the shared SDK emitter. We attach a fresh
@@ -2689,6 +2689,73 @@ export declare class NeuroLink {
      * `decisionHooks` and emits `decision:before` / `decision:after`.
      */
     tryDecide(options: DecisionCallerOptions): Promise<DecisionResult | null>;
+    /**
+     * Transcribe a recording to text through any STT provider, with the
+     * optional layers that make self-hosted engines usable: a dictionary of
+     * names and jargon (sent to the engine as context, then repaired after the
+     * fact behind a `tryDecide` guard), an LLM rewrite, a second-opinion
+     * engine, and a fallback engine for audio the primary cannot read.
+     *
+     * The provider defaults to `NEUROLINK_STT_PROVIDER`, then the first
+     * configured STT provider, then the built-in local engine, so an instance
+     * with no STT configuration still transcribes. Per-call
+     * `credentials.stt` beats the instance's, which beats the environment.
+     *
+     * @example
+     * ```typescript
+     * const result = await neurolink.transcribe({
+     *   audio: "./meeting.wav",
+     *   dictionary: [{ term: "NeuroLink", heardAs: ["neural link"] }],
+     * });
+     * console.log(result.text, result.engine.provider, result.steps);
+     * ```
+     *
+     * @throws STTError when the audio cannot be read, the provider is unknown,
+     *   or the engine fails with no fallback that covers errors
+     */
+    transcribe(options: TranscribeOptions): Promise<TranscribeResult>;
+    /**
+     * Transcribe a live stream of PCM16LE frames. Engines with a native stream
+     * use it; every other engine is streamed through the chunked adapter
+     * (energy-gated utterances, rolling re-transcription, LocalAgreement
+     * commits). Emits `interim`, `final`, `corrected` and the other
+     * {@link TranscribeStreamEvent}s; the same correction, fallback and
+     * provider rules as {@link NeuroLink.transcribe} apply per utterance.
+     *
+     * @example
+     * ```typescript
+     * for await (const event of neurolink.transcribeStream({ audio: micFrames })) {
+     *   if (event.type === "final") console.log(event.text);
+     * }
+     * ```
+     */
+    transcribeStream(options: TranscribeStreamOptions): AsyncIterable<TranscribeStreamEvent>;
+    /**
+     * Instance `credentials.stt` with a per-call `stt` slice merged on top,
+     * provider by provider and field by field (a per-call `apiKey` keeps the
+     * instance's `baseURL`). Undefined when neither has one.
+     */
+    private resolveSTTCredentials;
+    /**
+     * The STT provider for a call: an explicit name (or alias) as its
+     * canonical name, a name only a caller-registered handler knows as is, and
+     * no name as the default provider for these credentials.
+     */
+    private resolveSTTProvider;
+    /**
+     * Everything the transcribe orchestration needs from this instance: STT
+     * handlers with the request's credentials applied, provider resolution,
+     * the fail-open decision model and a rewrite model.
+     */
+    private buildTranscribeDeps;
+    /**
+     * The correction layer's two model calls: `decide` is {@link tryDecide}
+     * (fail-open — `null` on any failure), `rewrite` a plain text call with
+     * tools and memory off (`onPartial`, when given, receives the finished
+     * text once — see the note inside on why it is not streamed) and aborted
+     * at its timeout.
+     */
+    private buildSTTCorrectionDeps;
     /**
      * `tryDecide` without the site handling: the try/catch around the call.
      * Dispatches through `this.decide`, so a subclass override or an instance

package/dist/neurolink.js

@@ -31,6 +31,7 @@ import { ToolRoutingCache } from "./core/toolRoutingCache.js";
 import { DEFAULT_RECENT_TURNS, KnowledgeGroundingEngine, } from "./knowledge/index.js";
 import { AIProviderFactory } from "./core/factory.js";
 import { describeDecisionProviderKeys, PROVIDER_ALIAS_INDEX, PROVIDER_DESCRIPTORS_BY_NAME, resolveDefaultDecisionProvider, } from "./factories/providerDescriptors.js";
+import { describeSTTProviderKeys, getSTTProviderDescriptor, listConfiguredSTTProviders, resolveCallSTTEndpoints, resolveDefaultSTTProvider, resolveSTTProviderName, } from "./factories/sttDescriptors.js";
 import { MAX_PARALLEL_DECISION_BATCHES, mergeDecisionResults, splitDecisionRequest, } from "./utils/decisionBatches.js";
 import { decisionKey, HOST_DECISION_NAMESPACE, isHostDecisionKey, snapshotDecisionQuestion, stripHostAnswers, } from "./utils/decisionAnswers.js";
 import { resolveDecisionLimitsReading } from "./utils/decisionLimits.js";
@@ -4238,8 +4239,9 @@ Current user's request: ${currentInput}`;
                 // registration failed silently after AI providers were registered.
                 await ProviderRegistry.registerAllProviders();
                 const { STTProcessor } = await import("./utils/sttProcessor.js");
-                const sttProvider = options.stt.provider ?? "whisper";
-                sttTranscription = await STTProcessor.transcribe(options.stt.audio, sttProvider, options.stt);
+                const sttCredentials = this.resolveSTTCredentials(options.credentials?.stt);
+                const sttProvider = this.resolveSTTProvider(options.stt.provider, sttCredentials);
+                sttTranscription = await STTProcessor.transcribe(options.stt.audio, sttProvider, options.stt, sttCredentials);
                 // Inject transcription into the LLM prompt
                 if (sttTranscription.text) {
                     const existingText = textOptions.prompt || textOptions.input?.text || "";
@@ -7606,8 +7608,9 @@ Current user's request: ${currentInput}`;
                     // registerAllProviders() is idempotent; always call.
                     await ProviderRegistry.registerAllProviders();
                     const { STTProcessor } = await import("./utils/sttProcessor.js");
-                    const sttProvider = sttOptions.provider ?? "whisper";
-                    streamSttTranscription = await STTProcessor.transcribe(sttAudio, sttProvider, sttOptions);
+                    const sttCredentials = this.resolveSTTCredentials(options.credentials?.stt);
+                    const sttProvider = this.resolveSTTProvider(sttOptions.provider, sttCredentials);
+                    streamSttTranscription = await STTProcessor.transcribe(sttAudio, sttProvider, sttOptions, sttCredentials);
                     if (streamSttTranscription.text) {
                         const existingText = options.input.text || "";
                         options.input.text = existingText
@@ -13979,6 +13982,201 @@ Current user's request: ${currentInput}`;
     async tryDecide(options) {
         return this.siteDecide(options);
     }
+    // ==========================================================================
+    // TRANSCRIBE — the speech-to-text inference type
+    // ==========================================================================
+    /**
+     * Transcribe a recording to text through any STT provider, with the
+     * optional layers that make self-hosted engines usable: a dictionary of
+     * names and jargon (sent to the engine as context, then repaired after the
+     * fact behind a `tryDecide` guard), an LLM rewrite, a second-opinion
+     * engine, and a fallback engine for audio the primary cannot read.
+     *
+     * The provider defaults to `NEUROLINK_STT_PROVIDER`, then the first
+     * configured STT provider, then the built-in local engine, so an instance
+     * with no STT configuration still transcribes. Per-call
+     * `credentials.stt` beats the instance's, which beats the environment.
+     *
+     * @example
+     * ```typescript
+     * const result = await neurolink.transcribe({
+     *   audio: "./meeting.wav",
+     *   dictionary: [{ term: "NeuroLink", heardAs: ["neural link"] }],
+     * });
+     * console.log(result.text, result.engine.provider, result.steps);
+     * ```
+     *
+     * @throws STTError when the audio cannot be read, the provider is unknown,
+     *   or the engine fails with no fallback that covers errors
+     */
+    async transcribe(options) {
+        const [deps, { runTranscribe }] = await Promise.all([
+            this.buildTranscribeDeps(options.credentials?.stt),
+            import("./voice/transcribe.js"),
+        ]);
+        return runTranscribe(options, deps);
+    }
+    /**
+     * Transcribe a live stream of PCM16LE frames. Engines with a native stream
+     * use it; every other engine is streamed through the chunked adapter
+     * (energy-gated utterances, rolling re-transcription, LocalAgreement
+     * commits). Emits `interim`, `final`, `corrected` and the other
+     * {@link TranscribeStreamEvent}s; the same correction, fallback and
+     * provider rules as {@link NeuroLink.transcribe} apply per utterance.
+     *
+     * @example
+     * ```typescript
+     * for await (const event of neurolink.transcribeStream({ audio: micFrames })) {
+     *   if (event.type === "final") console.log(event.text);
+     * }
+     * ```
+     */
+    async *transcribeStream(options) {
+        const [deps, { runTranscribeStream }] = await Promise.all([
+            this.buildTranscribeDeps(options.credentials?.stt),
+            import("./voice/transcribeStream.js"),
+        ]);
+        yield* runTranscribeStream(options, deps);
+    }
+    /**
+     * Instance `credentials.stt` with a per-call `stt` slice merged on top,
+     * provider by provider and field by field (a per-call `apiKey` keeps the
+     * instance's `baseURL`). Undefined when neither has one.
+     */
+    resolveSTTCredentials(callCredentials) {
+        const instance = this.credentials?.stt;
+        if (!instance || !callCredentials) {
+            return callCredentials ?? instance;
+        }
+        const merged = { ...instance };
+        for (const [key, callSlice] of Object.entries(callCredentials)) {
+            const instanceSlice = merged[key];
+            merged[key] =
+                instanceSlice &&
+                    typeof instanceSlice === "object" &&
+                    callSlice &&
+                    typeof callSlice === "object"
+                    ? { ...instanceSlice, ...callSlice }
+                    : (callSlice ?? instanceSlice);
+        }
+        return merged;
+    }
+    /**
+     * The STT provider for a call: an explicit name (or alias) as its
+     * canonical name, a name only a caller-registered handler knows as is, and
+     * no name as the default provider for these credentials.
+     */
+    resolveSTTProvider(name, credentials) {
+        const trimmed = name?.trim();
+        if (!trimmed) {
+            return resolveDefaultSTTProvider(credentials);
+        }
+        return resolveSTTProviderName(trimmed) ?? trimmed;
+    }
+    /**
+     * Everything the transcribe orchestration needs from this instance: STT
+     * handlers with the request's credentials applied, provider resolution,
+     * the fail-open decision model and a rewrite model.
+     */
+    async buildTranscribeDeps(callCredentials) {
+        await ProviderRegistry.registerAllProviders();
+        const [{ STTProcessor }, { createSTTHandler }, { STTError }] = await Promise.all([
+            import("./utils/sttProcessor.js"),
+            import("./voice/index.js"),
+            import("./voice/errors.js"),
+        ]);
+        const credentials = this.resolveSTTCredentials(callCredentials);
+        const resolveProvider = (name) => {
+            const resolved = this.resolveSTTProvider(name, credentials);
+            if (resolveSTTProviderName(resolved) === undefined &&
+                !STTProcessor.supports(resolved) &&
+                !resolveCallSTTEndpoints(credentials)[resolved.trim().toLowerCase()]) {
+                throw STTError.unknownProvider(resolved, listConfiguredSTTProviders(credentials), describeSTTProviderKeys());
+            }
+            return resolved;
+        };
+        // Same precedence as STTProcessor.resolveHandler, synchronously: a
+        // credentials slice for the provider builds a handler from it; else the
+        // registered handler; else, for a shipped provider, one from the env.
+        const getHandler = (name) => {
+            if (resolveCallSTTEndpoints(credentials)[name.trim().toLowerCase()]) {
+                return createSTTHandler(name, credentials);
+            }
+            const descriptor = getSTTProviderDescriptor(name);
+            if (!descriptor) {
+                return STTProcessor.getHandler(name);
+            }
+            if (credentials?.[descriptor.credentialsKey] !== undefined) {
+                return createSTTHandler(descriptor.name, credentials);
+            }
+            return (STTProcessor.getHandler(descriptor.name) ??
+                createSTTHandler(descriptor.name));
+        };
+        return {
+            ...this.buildSTTCorrectionDeps(),
+            transcribe: (audio, provider, options) => STTProcessor.transcribe(audio, provider, options, credentials),
+            resolveProvider,
+            getHandler,
+        };
+    }
+    /**
+     * The correction layer's two model calls: `decide` is {@link tryDecide}
+     * (fail-open — `null` on any failure), `rewrite` a plain text call with
+     * tools and memory off (`onPartial`, when given, receives the finished
+     * text once — see the note inside on why it is not streamed) and aborted
+     * at its timeout.
+     */
+    buildSTTCorrectionDeps() {
+        return {
+            decide: async (o) => {
+                // The correction layer builds plain JSON states and question maps;
+                // the decision provider validates them on its side.
+                const result = await this.tryDecide({
+                    state: o.state,
+                    questions: o.questions,
+                    ...(o.timeoutMs !== undefined ? { timeoutMs: o.timeoutMs } : {}),
+                });
+                return result ? { answers: result.answers } : null;
+            },
+            rewrite: async (o) => {
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(), o.timeoutMs);
+                // A server runs many transcriptions with one rewrite model, so the
+                // model is configurable once, in the environment; a request's own
+                // `correction.rewrite.provider/model` still wins.
+                const provider = o.provider ?? process.env.NEUROLINK_STT_REWRITE_PROVIDER?.trim();
+                const model = o.model ?? process.env.NEUROLINK_STT_REWRITE_MODEL?.trim();
+                const request = {
+                    input: { text: o.user },
+                    systemPrompt: o.system,
+                    ...(provider ? { provider } : {}),
+                    ...(model ? { model } : {}),
+                    maxTokens: o.maxTokens,
+                    temperature: 0,
+                    disableTools: true,
+                    memory: { enabled: false },
+                    abortSignal: controller.signal,
+                };
+                try {
+                    // Always `generate()`. Measured on the same rewrite against the same
+                    // model: generate 0.7 s; stream() 3.4 s with the first chunk at
+                    // 3.3 s — the stream path's setup costs more than the whole answer
+                    // and delivers it in one burst anyway, so a "streamed" rewrite only
+                    // risked the 12 s timeout. `onPartial` still fires once with the
+                    // full text so a live caller replaces the sentence the same way.
+                    const generated = await this.generate(request);
+                    const text = generated.content ?? "";
+                    if (o.onPartial && text) {
+                        o.onPartial(text);
+                    }
+                    return text;
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+            },
+        };
+    }
     /**
      * `tryDecide` without the site handling: the try/catch around the call.
      * Dispatches through `this.decide`, so a subclass override or an instance

package/dist/processors/media/AudioProcessor.d.ts

@@ -90,28 +90,22 @@ export declare class AudioProcessor extends BaseFileProcessor<ProcessedAudio> {
      */
     processFile(fileInfo: FileInfo, options?: ProcessOptions & AudioProcessorOptions): Promise<ProcessorFileProcessingResult<ProcessedAudio>>;
     /**
-     * Whether a transcription backend has the credentials it needs (#413).
+     * Choose the transcription backend for this file (#413).
+     *
+     * With no `requested` backend, the STT default applies: the provider named
+     * by `NEUROLINK_STT_PROVIDER`, else the first configured STT provider in
+     * descriptor order, else the built-in local engine when it can run here.
+     * With one, it is resolved through the legacy names and the STT aliases and
+     * validated for availability — a backend the caller explicitly asked for is
+     * never silently swapped for a different one, because a caller who pinned
+     * Azure for a data-residency reason would not want OpenAI chosen behind
+     * their back. The mismatch is reported instead.
      *
      * Availability is decided from the environment only — no network call — so
      * selection stays cheap and cannot itself fail. A key that is present but
      * rejected upstream surfaces later, as a transcription failure with the
      * provider's own message, not as "unavailable".
      *
-     * @param provider - Backend to check
-     * @returns True when at least one of the backend's credential vars is set
-     */
-    private isProviderAvailable;
-    /**
-     * Choose the transcription backend for this file (#413).
-     *
-     * With no `requested` backend, the first configured entry of
-     * {@link TRANSCRIPTION_PROVIDER_ORDER} wins (OpenAI, then Google, then
-     * Azure). With one, it is normalised through the alias table and validated
-     * for availability — a backend the caller explicitly asked for is never
-     * silently swapped for a different one, because a caller who pinned Azure
-     * for a data-residency reason would not want OpenAI chosen behind their
-     * back. The mismatch is reported instead.
-     *
      * @param requested - Caller's `AudioProcessorOptions.provider`, if any
      * @returns The chosen backend, or the reason no backend could be chosen
      */
@@ -120,10 +114,14 @@ export declare class AudioProcessor extends BaseFileProcessor<ProcessedAudio> {
      * Attempt to transcribe audio with whichever backend is configured (#413).
      *
      * Transcription is attempted when:
-     * 1. A backend's credentials are present (see {@link selectProvider})
+     * 1. A backend is available (see {@link selectProvider})
      * 2. File size is within the 25MB ceiling
      * 3. The file format is one the backend accepts
      *
+     * The call itself goes through `STTProcessor`, the same path as
+     * `transcribe()` and `generate({ stt })`, so `OPENAI_STT_*`, the default
+     * provider rule and every STT handler apply here too.
+     *
      * Gracefully degrades: if transcription fails for any reason, metadata-only
      * output is returned (transcription is additive, never blocks processing).
      *
@@ -134,45 +132,6 @@ export declare class AudioProcessor extends BaseFileProcessor<ProcessedAudio> {
      * @returns Transcription result with transcript text, or the reason there is none
      */
     private attemptTranscription;
-    /**
-     * Transcribe via OpenAI Whisper (#416).
-     *
-     * A native multipart POST to OpenAI's transcription endpoint. This used to
-     * go through @ai-sdk/openai's createOpenAI().transcription() plus the ai
-     * package's experimental_transcribe; both were dropped, and this is the
-     * only wire behaviour of theirs the processor ever depended on. The same
-     * request is already made natively by voice/providers/OpenAISTT.ts.
-     *
-     * `verbose_json` is requested for `language` and `duration` alongside the
-     * text — the response was previously parsed for `text` alone, so both were
-     * received and discarded, and `FileProcessingResult.metadata.language` had
-     * nothing to report (#409).
-     *
-     * @param buffer - Audio file content
-     * @param filename - Original filename, sent as the multipart part name
-     * @param mimetype - MIME type used for the upload blob
-     * @param options - Caller's language / model / prompt overrides
-     * @param skipped - Builds the "no transcript, and here is why" outcome
-     * @returns Transcription result, or the reason there is none
-     */
-    private transcribeWithOpenAI;
-    /**
-     * Transcribe via one of the non-OpenAI STT handlers (#413).
-     *
-     * Google and Azure already have working, tested implementations under
-     * `src/lib/voice/providers/`, reached here through a dynamic import so a
-     * file-processing run that never transcribes does not pay to load them —
-     * the same reason `music-metadata` is loaded lazily above.
-     *
-     * @param chosen - Backend and display label from {@link selectProvider}
-     * @param buffer - Audio file content
-     * @param filename - Original filename; its extension picks the wire format
-     * @param mimetype - MIME type, used as the fallback format signal
-     * @param options - Caller's language / model overrides
-     * @param skipped - Builds the "no transcript, and here is why" outcome
-     * @returns Transcription result, or the reason there is none
-     */
-    private transcribeWithHandler;
     /**
      * Stub implementation required by BaseFileProcessor.
      * Not used because processFile is fully overridden.