@juspay/neurolink 12.48.2 → 12.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/browser/neurolink.min.js +616 -595
- package/dist/cli/commands/serve.d.ts +0 -3
- package/dist/cli/commands/serve.js +85 -3
- package/dist/cli/commands/transcribe.d.ts +24 -0
- package/dist/cli/commands/transcribe.js +718 -0
- package/dist/cli/factories/commandFactory.d.ts +10 -0
- package/dist/cli/factories/commandFactory.js +13 -0
- package/dist/cli/parser.js +3 -0
- package/dist/factories/mediaHandlerCatalog.js +11 -9
- package/dist/factories/sttDescriptors.d.ts +63 -0
- package/dist/factories/sttDescriptors.js +286 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +7 -1
- package/dist/neurolink.d.ts +68 -1
- package/dist/neurolink.js +202 -4
- package/dist/processors/media/AudioProcessor.d.ts +15 -56
- package/dist/processors/media/AudioProcessor.js +105 -276
- package/dist/processors/media/VideoProcessor.d.ts +3 -5
- package/dist/processors/media/VideoProcessor.js +52 -51
- package/dist/server/abstract/baseServerAdapter.d.ts +7 -0
- package/dist/server/abstract/baseServerAdapter.js +13 -1
- package/dist/server/adapters/expressAdapter.d.ts +1 -0
- package/dist/server/adapters/expressAdapter.js +48 -4
- package/dist/server/adapters/fastifyAdapter.d.ts +1 -0
- package/dist/server/adapters/fastifyAdapter.js +38 -1
- package/dist/server/adapters/honoAdapter.d.ts +1 -4
- package/dist/server/adapters/honoAdapter.js +83 -1
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +43 -0
- package/dist/server/index.d.ts +3 -2
- package/dist/server/index.js +3 -2
- package/dist/server/routes/index.d.ts +1 -0
- package/dist/server/routes/index.js +3 -0
- package/dist/server/routes/transcribeRoutes.d.ts +23 -0
- package/dist/server/routes/transcribeRoutes.js +506 -0
- package/dist/server/utils/rawBody.d.ts +32 -0
- package/dist/server/utils/rawBody.js +76 -0
- package/dist/server/utils/validation.d.ts +192 -0
- package/dist/server/utils/validation.js +100 -0
- package/dist/server/websocket/transcribeWebSocket.d.ts +42 -0
- package/dist/server/websocket/transcribeWebSocket.js +264 -0
- package/dist/types/cli.d.ts +46 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/providers.d.ts +7 -1
- package/dist/types/server.d.ts +117 -1
- package/dist/types/stt.d.ts +160 -7
- package/dist/types/transcribe.d.ts +506 -0
- package/dist/types/transcribe.js +29 -0
- package/dist/utils/speakerTurns.d.ts +28 -0
- package/dist/utils/speakerTurns.js +93 -0
- package/dist/utils/sttProcessor.d.ts +20 -2
- package/dist/utils/sttProcessor.js +73 -5
- package/dist/voice/correction/dictionary.d.ts +49 -0
- package/dist/voice/correction/dictionary.js +134 -0
- package/dist/voice/correction/guard.d.ts +35 -0
- package/dist/voice/correction/guard.js +161 -0
- package/dist/voice/correction/index.d.ts +47 -0
- package/dist/voice/correction/index.js +211 -0
- package/dist/voice/correction/rewrite.d.ts +92 -0
- package/dist/voice/correction/rewrite.js +322 -0
- package/dist/voice/errors.d.ts +15 -0
- package/dist/voice/errors.js +45 -0
- package/dist/voice/index.d.ts +20 -2
- package/dist/voice/index.js +75 -9
- package/dist/voice/providers/AzureSTT.d.ts +10 -7
- package/dist/voice/providers/AzureSTT.js +23 -75
- package/dist/voice/providers/DeepgramSTT.d.ts +17 -2
- package/dist/voice/providers/DeepgramSTT.js +109 -13
- package/dist/voice/providers/ElevenLabsSTT.d.ts +12 -2
- package/dist/voice/providers/ElevenLabsSTT.js +34 -8
- package/dist/voice/providers/GoogleSTT.d.ts +10 -8
- package/dist/voice/providers/GoogleSTT.js +21 -75
- package/dist/voice/providers/OpenAISTT.d.ts +30 -9
- package/dist/voice/providers/OpenAISTT.js +196 -21
- package/dist/voice/providers/WhistleSTT.d.ts +87 -0
- package/dist/voice/providers/WhistleSTT.js +313 -0
- package/dist/voice/streaming/chunkedStreamAdapter.d.ts +19 -0
- package/dist/voice/streaming/chunkedStreamAdapter.js +519 -0
- package/dist/voice/streaming/endpointer.d.ts +81 -0
- package/dist/voice/streaming/endpointer.js +247 -0
- package/dist/voice/streaming/localAgreement.d.ts +30 -0
- package/dist/voice/streaming/localAgreement.js +44 -0
- package/dist/voice/sttEnvCredentials.d.ts +12 -0
- package/dist/voice/sttEnvCredentials.js +12 -0
- package/dist/voice/transcribe.d.ts +47 -0
- package/dist/voice/transcribe.js +519 -0
- package/dist/voice/transcribeStream.d.ts +4 -0
- package/dist/voice/transcribeStream.js +584 -0
- package/dist/voice/whistle/assets.d.ts +47 -0
- package/dist/voice/whistle/assets.js +270 -0
- package/dist/voice/whistle/audio.d.ts +32 -0
- package/dist/voice/whistle/audio.js +267 -0
- package/dist/voice/whistle/engine.d.ts +32 -0
- package/dist/voice/whistle/engine.js +193 -0
- package/dist/voice/whistle/worker.d.ts +18 -0
- package/dist/voice/whistle/worker.js +294 -0
- package/docs-site/static/search-index.json +66 -8
- package/models/whistle/README.md +18 -0
- package/models/whistle/needle.js +2 -0
- package/models/whistle/needle.wasm +0 -0
- package/models/whistle/whistle.cact +0 -0
- package/package.json +7 -3
|
@@ -300,6 +300,11 @@ export declare class CLICommandFactory {
|
|
|
300
300
|
type: "string";
|
|
301
301
|
description: string;
|
|
302
302
|
};
|
|
303
|
+
sttDiarize: {
|
|
304
|
+
type: "boolean";
|
|
305
|
+
default: boolean;
|
|
306
|
+
description: string;
|
|
307
|
+
};
|
|
303
308
|
inputAudio: {
|
|
304
309
|
type: "string";
|
|
305
310
|
description: string;
|
|
@@ -1223,6 +1228,11 @@ export declare const commonOptions: {
|
|
|
1223
1228
|
type: "string";
|
|
1224
1229
|
description: string;
|
|
1225
1230
|
};
|
|
1231
|
+
sttDiarize: {
|
|
1232
|
+
type: "boolean";
|
|
1233
|
+
default: boolean;
|
|
1234
|
+
description: string;
|
|
1235
|
+
};
|
|
1226
1236
|
inputAudio: {
|
|
1227
1237
|
type: "string";
|
|
1228
1238
|
description: string;
|
|
@@ -435,6 +435,11 @@ export class CLICommandFactory {
|
|
|
435
435
|
type: "string",
|
|
436
436
|
description: "Audio language code for STT (e.g., en-US)",
|
|
437
437
|
},
|
|
438
|
+
sttDiarize: {
|
|
439
|
+
type: "boolean",
|
|
440
|
+
default: false,
|
|
441
|
+
description: "Label speakers in the transcript (providers that diarize: deepgram, google-stt, elevenlabs-stt, a diarizing OpenAI-compatible endpoint)",
|
|
442
|
+
},
|
|
438
443
|
inputAudio: {
|
|
439
444
|
type: "string",
|
|
440
445
|
description: "Path to audio file for STT transcription",
|
|
@@ -818,6 +823,7 @@ export class CLICommandFactory {
|
|
|
818
823
|
// --input-audio without --stt get an actionable error from yargs
|
|
819
824
|
// instead of silently skipping STT.
|
|
820
825
|
.implies("sttProvider", "stt")
|
|
826
|
+
.implies("sttDiarize", "stt")
|
|
821
827
|
.implies("inputAudio", "stt"));
|
|
822
828
|
}
|
|
823
829
|
// Helper method to process CLI images with smart auto-detection
|
|
@@ -980,6 +986,7 @@ export class CLICommandFactory {
|
|
|
980
986
|
stt: argv.stt,
|
|
981
987
|
sttProvider: argv.sttProvider,
|
|
982
988
|
sttLanguage: argv.sttLanguage,
|
|
989
|
+
sttDiarize: argv.sttDiarize,
|
|
983
990
|
inputAudio: argv.inputAudio,
|
|
984
991
|
// Video generation options (Veo 3.1)
|
|
985
992
|
outputMode: argv.outputMode,
|
|
@@ -2850,6 +2857,9 @@ export class CLICommandFactory {
|
|
|
2850
2857
|
enabled: true,
|
|
2851
2858
|
provider: enhancedOptions.sttProvider,
|
|
2852
2859
|
language: enhancedOptions.sttLanguage,
|
|
2860
|
+
...(enhancedOptions.sttDiarize
|
|
2861
|
+
? { speakerDiarization: true }
|
|
2862
|
+
: {}),
|
|
2853
2863
|
...(inputAudioBuffer && { audio: inputAudioBuffer }),
|
|
2854
2864
|
...(inputAudioFormat && { format: inputAudioFormat }),
|
|
2855
2865
|
}
|
|
@@ -3117,6 +3127,9 @@ export class CLICommandFactory {
|
|
|
3117
3127
|
enabled: true,
|
|
3118
3128
|
provider: enhancedOptions.sttProvider,
|
|
3119
3129
|
language: enhancedOptions.sttLanguage,
|
|
3130
|
+
...(enhancedOptions.sttDiarize
|
|
3131
|
+
? { speakerDiarization: true }
|
|
3132
|
+
: {}),
|
|
3120
3133
|
...(streamSttAudio && { audio: streamSttAudio }),
|
|
3121
3134
|
...(streamSttFormat && { format: streamSttFormat }),
|
|
3122
3135
|
};
|
package/dist/cli/parser.js
CHANGED
|
@@ -31,6 +31,7 @@ import { proxyExposeCommand } from "./commands/proxyExpose.js";
|
|
|
31
31
|
import { proxyReplayCommand } from "./commands/proxyReplay.js";
|
|
32
32
|
import { EvaluateCommandFactory } from "./commands/evaluate.js";
|
|
33
33
|
import { DecideCommandFactory } from "./commands/decide.js";
|
|
34
|
+
import { TranscribeCommandFactory } from "./commands/transcribe.js";
|
|
34
35
|
import { TaskCommandFactory } from "./commands/task.js";
|
|
35
36
|
import { AutoresearchCommandFactory } from "./commands/autoresearch.js";
|
|
36
37
|
import { voiceServerCommand } from "./commands/voiceServer.js";
|
|
@@ -172,6 +173,8 @@ export function initializeCliParser() {
|
|
|
172
173
|
.command(CLICommandFactory.createStreamCommand())
|
|
173
174
|
// Decide Command - typed, calibrated judgements (no free text)
|
|
174
175
|
.command(DecideCommandFactory.createDecideCommand())
|
|
176
|
+
// Transcribe Command - audio in, text out (the transcribe inference type)
|
|
177
|
+
.command(TranscribeCommandFactory.createTranscribeCommand())
|
|
175
178
|
// Batch Processing Command - Using CLICommandFactory
|
|
176
179
|
.command(CLICommandFactory.createBatchCommand())
|
|
177
180
|
// Provider Command Group - Using CLICommandFactory
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { STT_PROVIDER_DESCRIPTORS } from "./sttDescriptors.js";
|
|
1
2
|
/**
|
|
2
3
|
* Static catalog of every shipped media-handler provider, across all six
|
|
3
4
|
* ecosystems (TTS, STT, Realtime, Video, Avatar, Music). Pure data — no
|
|
@@ -20,15 +21,16 @@ export const MEDIA_HANDLER_CATALOG = [
|
|
|
20
21
|
{ kind: "tts", name: "sixtydb" },
|
|
21
22
|
{ kind: "tts", name: "cartesia" },
|
|
22
23
|
// --- STT ---
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
//
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
24
|
+
// Derived from STT_PROVIDER_DESCRIPTORS (names, aliases and precedence
|
|
25
|
+
// order) so the catalog and the descriptors cannot drift. "elevenlabs" is
|
|
26
|
+
// reused across kinds on purpose — TTS primary, STT alias — the same way
|
|
27
|
+
// "vertex" is a TTS alias and the Video primary; each kind has its own
|
|
28
|
+
// registry.
|
|
29
|
+
...STT_PROVIDER_DESCRIPTORS.map((descriptor) => ({
|
|
30
|
+
kind: "stt",
|
|
31
|
+
name: descriptor.name,
|
|
32
|
+
...(descriptor.aliases ? { aliases: descriptor.aliases } : {}),
|
|
33
|
+
})),
|
|
32
34
|
// --- Realtime ---
|
|
33
35
|
{ kind: "realtime", name: "openai-realtime" },
|
|
34
36
|
{ kind: "realtime", name: "gemini-live" },
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { STTCredentials, STTEndpointConfig, STTProviderDescriptor } from "../types/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Every shipped speech-to-text provider — the single source of truth for its
|
|
4
|
+
* name, aliases, credentials slice, env vars and capabilities. Pure data, like
|
|
5
|
+
* `PROVIDER_DESCRIPTORS` for text providers; the handler classes are wired to
|
|
6
|
+
* these names in `voice/index.ts`, and `MEDIA_HANDLER_CATALOG`'s STT rows are
|
|
7
|
+
* derived from this list so the two cannot drift.
|
|
8
|
+
*
|
|
9
|
+
* **Order is precedence.** `resolveDefaultSTTProvider()` returns the first
|
|
10
|
+
* entry that is configured, so reordering changes which engine every
|
|
11
|
+
* `transcribe()` / `generate({ stt })` without an explicit provider uses.
|
|
12
|
+
* `whistle` is last on purpose: it runs on this machine and needs no key, so
|
|
13
|
+
* it is what a caller gets when nothing else is configured.
|
|
14
|
+
*/
|
|
15
|
+
export declare const STT_PROVIDER_DESCRIPTORS: readonly STTProviderDescriptor[];
|
|
16
|
+
/** The provider used when nothing else is configured. */
|
|
17
|
+
export declare const DEFAULT_LOCAL_STT_PROVIDER = "whistle";
|
|
18
|
+
/**
|
|
19
|
+
* Canonical descriptor name for a provider name or alias (case-insensitive),
|
|
20
|
+
* or `undefined` when it is not a shipped STT provider — a caller may still
|
|
21
|
+
* have registered a handler of its own under that name.
|
|
22
|
+
*/
|
|
23
|
+
export declare function resolveSTTProviderName(name: string | undefined): string | undefined;
|
|
24
|
+
/** The descriptor for a provider name or alias, if it is a shipped one. */
|
|
25
|
+
export declare function getSTTProviderDescriptor(name: string): STTProviderDescriptor | undefined;
|
|
26
|
+
/**
|
|
27
|
+
* Whether a provider can run with these credentials and the environment. The
|
|
28
|
+
* local engine needs no configuration and always counts. Unknown names are
|
|
29
|
+
* not configured.
|
|
30
|
+
*/
|
|
31
|
+
export declare function isSTTProviderConfigured(name: string, credentials?: STTCredentials): boolean;
|
|
32
|
+
/** Configured providers, in precedence order (the local engine last). */
|
|
33
|
+
export declare function listConfiguredSTTProviders(credentials?: STTCredentials): string[];
|
|
34
|
+
/**
|
|
35
|
+
* Named OpenAI-compatible endpoints: `credentials.stt.endpoints` over the
|
|
36
|
+
* `NEUROLINK_STT_ENDPOINTS` JSON object. Keys are lowercased; one that
|
|
37
|
+
* collides with a shipped provider name or alias is dropped with a warning,
|
|
38
|
+
* as is an entry without a `baseURL`. Malformed JSON is reported once and
|
|
39
|
+
* ignored.
|
|
40
|
+
*/
|
|
41
|
+
export declare function resolveSTTEndpoints(credentials?: STTCredentials): Record<string, STTEndpointConfig>;
|
|
42
|
+
/**
|
|
43
|
+
* The per-call endpoints alone (`credentials.stt.endpoints`), validated
|
|
44
|
+
* exactly as the env map is: names lower-cased, a name that collides with a
|
|
45
|
+
* shipped provider or alias dropped, an entry without a `baseURL` dropped.
|
|
46
|
+
* Every per-call lookup goes through this so a request can neither replace a
|
|
47
|
+
* shipped handler nor build an endpoint the env form would have refused.
|
|
48
|
+
*/
|
|
49
|
+
export declare function resolveCallSTTEndpoints(credentials?: STTCredentials): Record<string, STTEndpointConfig>;
|
|
50
|
+
/**
|
|
51
|
+
* The provider a transcription without an explicit one goes to:
|
|
52
|
+
* `NEUROLINK_STT_PROVIDER` when it names a known provider or alias, else the
|
|
53
|
+
* first configured provider in descriptor order, else the built-in local
|
|
54
|
+
* engine. Never throws, never returns an empty string.
|
|
55
|
+
*/
|
|
56
|
+
export declare function resolveDefaultSTTProvider(credentials?: STTCredentials): string;
|
|
57
|
+
/**
|
|
58
|
+
* The variables that would configure each hosted STT provider, one clause per
|
|
59
|
+
* provider in precedence order, e.g. "OPENAI_STT_API_KEY or OPENAI_API_KEY for
|
|
60
|
+
* whisper, or DEEPGRAM_API_KEY for deepgram". Derived from the descriptors so
|
|
61
|
+
* a new provider appears in every "nothing is configured" message.
|
|
62
|
+
*/
|
|
63
|
+
export declare function describeSTTProviderKeys(): string;
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import { logger } from "../utils/logger.js";
|
|
2
|
+
/**
|
|
3
|
+
* Every shipped speech-to-text provider — the single source of truth for its
|
|
4
|
+
* name, aliases, credentials slice, env vars and capabilities. Pure data, like
|
|
5
|
+
* `PROVIDER_DESCRIPTORS` for text providers; the handler classes are wired to
|
|
6
|
+
* these names in `voice/index.ts`, and `MEDIA_HANDLER_CATALOG`'s STT rows are
|
|
7
|
+
* derived from this list so the two cannot drift.
|
|
8
|
+
*
|
|
9
|
+
* **Order is precedence.** `resolveDefaultSTTProvider()` returns the first
|
|
10
|
+
* entry that is configured, so reordering changes which engine every
|
|
11
|
+
* `transcribe()` / `generate({ stt })` without an explicit provider uses.
|
|
12
|
+
* `whistle` is last on purpose: it runs on this machine and needs no key, so
|
|
13
|
+
* it is what a caller gets when nothing else is configured.
|
|
14
|
+
*/
|
|
15
|
+
export const STT_PROVIDER_DESCRIPTORS = [
|
|
16
|
+
{
|
|
17
|
+
name: "whisper",
|
|
18
|
+
aliases: ["openai-stt", "openai-compatible"],
|
|
19
|
+
label: "OpenAI-compatible /audio/transcriptions",
|
|
20
|
+
credentialsKey: "whisper",
|
|
21
|
+
envVars: ["OPENAI_STT_API_KEY", "OPENAI_API_KEY"],
|
|
22
|
+
baseUrlEnv: "OPENAI_STT_BASE_URL",
|
|
23
|
+
defaultModel: "whisper-1",
|
|
24
|
+
capabilities: {
|
|
25
|
+
streaming: "chunked",
|
|
26
|
+
// OpenAI's own endpoint does not diarize; compatible servers may, and
|
|
27
|
+
// the handler only sends the switch to those.
|
|
28
|
+
diarization: true,
|
|
29
|
+
languageDetect: true,
|
|
30
|
+
prompt: true,
|
|
31
|
+
wordTimestamps: true,
|
|
32
|
+
local: false,
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
name: "deepgram",
|
|
37
|
+
label: "Deepgram",
|
|
38
|
+
credentialsKey: "deepgram",
|
|
39
|
+
envVars: ["DEEPGRAM_API_KEY"],
|
|
40
|
+
baseUrlEnv: "DEEPGRAM_BASE_URL",
|
|
41
|
+
defaultModel: "nova-2",
|
|
42
|
+
capabilities: {
|
|
43
|
+
streaming: "native",
|
|
44
|
+
diarization: true,
|
|
45
|
+
languageDetect: true,
|
|
46
|
+
prompt: false,
|
|
47
|
+
wordTimestamps: true,
|
|
48
|
+
local: false,
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
name: "elevenlabs-stt",
|
|
53
|
+
// "elevenlabs" is reused across kinds on purpose — TTS primary, STT
|
|
54
|
+
// alias. Each kind has its own registry, so `stt: { provider:
|
|
55
|
+
// "elevenlabs" }` and `tts: { provider: "elevenlabs" }` resolve to
|
|
56
|
+
// different handlers.
|
|
57
|
+
aliases: ["scribe", "elevenlabs"],
|
|
58
|
+
label: "ElevenLabs Scribe",
|
|
59
|
+
credentialsKey: "elevenlabs",
|
|
60
|
+
envVars: ["ELEVENLABS_API_KEY"],
|
|
61
|
+
baseUrlEnv: "ELEVENLABS_STT_BASE_URL",
|
|
62
|
+
defaultModel: "scribe_v2",
|
|
63
|
+
capabilities: {
|
|
64
|
+
streaming: "chunked",
|
|
65
|
+
diarization: true,
|
|
66
|
+
languageDetect: true,
|
|
67
|
+
prompt: false,
|
|
68
|
+
wordTimestamps: true,
|
|
69
|
+
local: false,
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
name: "google-stt",
|
|
74
|
+
label: "Google Cloud Speech-to-Text",
|
|
75
|
+
credentialsKey: "google",
|
|
76
|
+
// GoogleSTT accepts the AI Studio key aliases and a service-account file.
|
|
77
|
+
envVars: [
|
|
78
|
+
"GOOGLE_API_KEY",
|
|
79
|
+
"GOOGLE_AI_API_KEY",
|
|
80
|
+
"GEMINI_API_KEY",
|
|
81
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
82
|
+
],
|
|
83
|
+
defaultModel: "latest_long",
|
|
84
|
+
capabilities: {
|
|
85
|
+
streaming: "chunked",
|
|
86
|
+
diarization: true,
|
|
87
|
+
languageDetect: false,
|
|
88
|
+
prompt: false,
|
|
89
|
+
wordTimestamps: true,
|
|
90
|
+
local: false,
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
name: "azure-stt",
|
|
95
|
+
label: "Azure Speech",
|
|
96
|
+
credentialsKey: "azure",
|
|
97
|
+
envVars: ["AZURE_SPEECH_KEY"],
|
|
98
|
+
capabilities: {
|
|
99
|
+
streaming: "chunked",
|
|
100
|
+
diarization: false,
|
|
101
|
+
languageDetect: false,
|
|
102
|
+
prompt: false,
|
|
103
|
+
wordTimestamps: true,
|
|
104
|
+
local: false,
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
name: "whistle",
|
|
109
|
+
aliases: ["cactus-whistle", "local"],
|
|
110
|
+
label: "Whistle (built-in, local)",
|
|
111
|
+
credentialsKey: "whistle",
|
|
112
|
+
envVars: [],
|
|
113
|
+
capabilities: {
|
|
114
|
+
streaming: "chunked",
|
|
115
|
+
diarization: false,
|
|
116
|
+
languageDetect: true,
|
|
117
|
+
prompt: false,
|
|
118
|
+
wordTimestamps: true,
|
|
119
|
+
local: true,
|
|
120
|
+
languages: ["en", "de", "fr", "es", "it", "nl", "pl"],
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
];
|
|
124
|
+
/** The provider used when nothing else is configured. */
|
|
125
|
+
export const DEFAULT_LOCAL_STT_PROVIDER = "whistle";
|
|
126
|
+
const STT_DESCRIPTORS_BY_NAME = new Map(STT_PROVIDER_DESCRIPTORS.map((d) => [d.name, d]));
|
|
127
|
+
/** Lower-cased name or alias → canonical descriptor name. */
|
|
128
|
+
const STT_ALIAS_INDEX = new Map(STT_PROVIDER_DESCRIPTORS.flatMap((d) => [
|
|
129
|
+
[d.name.toLowerCase(), d.name],
|
|
130
|
+
...(d.aliases ?? []).map((alias) => [alias.toLowerCase(), d.name]),
|
|
131
|
+
]));
|
|
132
|
+
/**
|
|
133
|
+
* Canonical descriptor name for a provider name or alias (case-insensitive),
|
|
134
|
+
* or `undefined` when it is not a shipped STT provider — a caller may still
|
|
135
|
+
* have registered a handler of its own under that name.
|
|
136
|
+
*/
|
|
137
|
+
export function resolveSTTProviderName(name) {
|
|
138
|
+
if (!name) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
return STT_ALIAS_INDEX.get(name.trim().toLowerCase());
|
|
142
|
+
}
|
|
143
|
+
/** The descriptor for a provider name or alias, if it is a shipped one. */
|
|
144
|
+
export function getSTTProviderDescriptor(name) {
|
|
145
|
+
const canonical = resolveSTTProviderName(name);
|
|
146
|
+
return canonical ? STT_DESCRIPTORS_BY_NAME.get(canonical) : undefined;
|
|
147
|
+
}
|
|
148
|
+
function hasValue(value) {
|
|
149
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Whether a credentials slice configures its provider: a key, a service
|
|
153
|
+
* account file, or — for the OpenAI-compatible handler, whose self-hosted
|
|
154
|
+
* servers often need no key — a base URL.
|
|
155
|
+
*/
|
|
156
|
+
function sliceConfigures(descriptor, credentials) {
|
|
157
|
+
const slice = credentials?.[descriptor.credentialsKey];
|
|
158
|
+
if (!slice) {
|
|
159
|
+
return false;
|
|
160
|
+
}
|
|
161
|
+
if (hasValue(slice.apiKey) || hasValue(slice.credentialsPath)) {
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
return descriptor.credentialsKey === "whisper" && hasValue(slice.baseURL);
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Whether a provider can run with these credentials and the environment. The
|
|
168
|
+
* local engine needs no configuration and always counts. Unknown names are
|
|
169
|
+
* not configured.
|
|
170
|
+
*/
|
|
171
|
+
export function isSTTProviderConfigured(name, credentials) {
|
|
172
|
+
const descriptor = getSTTProviderDescriptor(name);
|
|
173
|
+
if (!descriptor) {
|
|
174
|
+
return false;
|
|
175
|
+
}
|
|
176
|
+
if (descriptor.capabilities.local) {
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
if (sliceConfigures(descriptor, credentials)) {
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
return descriptor.envVars.some((envVar) => hasValue(process.env[envVar]));
|
|
183
|
+
}
|
|
184
|
+
/** Configured providers, in precedence order (the local engine last). */
|
|
185
|
+
export function listConfiguredSTTProviders(credentials) {
|
|
186
|
+
return [
|
|
187
|
+
...STT_PROVIDER_DESCRIPTORS.filter((d) => isSTTProviderConfigured(d.name, credentials)).map((d) => d.name),
|
|
188
|
+
...Object.keys(resolveSTTEndpoints(credentials)),
|
|
189
|
+
];
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Named OpenAI-compatible endpoints: `credentials.stt.endpoints` over the
|
|
193
|
+
* `NEUROLINK_STT_ENDPOINTS` JSON object. Keys are lowercased; one that
|
|
194
|
+
* collides with a shipped provider name or alias is dropped with a warning,
|
|
195
|
+
* as is an entry without a `baseURL`. Malformed JSON is reported once and
|
|
196
|
+
* ignored.
|
|
197
|
+
*/
|
|
198
|
+
export function resolveSTTEndpoints(credentials) {
|
|
199
|
+
const out = {};
|
|
200
|
+
const raw = process.env.NEUROLINK_STT_ENDPOINTS?.trim();
|
|
201
|
+
if (raw) {
|
|
202
|
+
try {
|
|
203
|
+
Object.assign(out, validateSTTEndpoints("NEUROLINK_STT_ENDPOINTS", JSON.parse(raw)));
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
logger.warn(`[STT] NEUROLINK_STT_ENDPOINTS is not valid JSON; ignoring it (${error instanceof Error ? error.message : String(error)})`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
Object.assign(out, resolveCallSTTEndpoints(credentials));
|
|
210
|
+
return out;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* The per-call endpoints alone (`credentials.stt.endpoints`), validated
|
|
214
|
+
* exactly as the env map is: names lower-cased, a name that collides with a
|
|
215
|
+
* shipped provider or alias dropped, an entry without a `baseURL` dropped.
|
|
216
|
+
* Every per-call lookup goes through this so a request can neither replace a
|
|
217
|
+
* shipped handler nor build an endpoint the env form would have refused.
|
|
218
|
+
*/
|
|
219
|
+
export function resolveCallSTTEndpoints(credentials) {
|
|
220
|
+
return validateSTTEndpoints("credentials.stt.endpoints", credentials?.endpoints);
|
|
221
|
+
}
|
|
222
|
+
function validateSTTEndpoints(source, entries) {
|
|
223
|
+
const out = {};
|
|
224
|
+
if (typeof entries !== "object" || entries === null) {
|
|
225
|
+
return out;
|
|
226
|
+
}
|
|
227
|
+
for (const [rawName, cfg] of Object.entries(entries)) {
|
|
228
|
+
const name = rawName.trim().toLowerCase();
|
|
229
|
+
if (!name) {
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (STT_ALIAS_INDEX.has(name)) {
|
|
233
|
+
logger.warn(`[STT] ${source}: endpoint "${rawName}" collides with a shipped provider name; ignoring it`);
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
const baseURL = typeof cfg === "object" && cfg !== null && "baseURL" in cfg
|
|
237
|
+
? cfg.baseURL
|
|
238
|
+
: undefined;
|
|
239
|
+
if (typeof baseURL !== "string" || !baseURL.trim()) {
|
|
240
|
+
logger.warn(`[STT] ${source}: endpoint "${rawName}" has no baseURL; ignoring it`);
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
const c = cfg;
|
|
244
|
+
out[name] = {
|
|
245
|
+
baseURL: baseURL.trim(),
|
|
246
|
+
...(typeof c.apiKey === "string" ? { apiKey: c.apiKey } : {}),
|
|
247
|
+
...(typeof c.model === "string" ? { model: c.model } : {}),
|
|
248
|
+
...(typeof c.timeoutMs === "number" ? { timeoutMs: c.timeoutMs } : {}),
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
return out;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* The provider a transcription without an explicit one goes to:
|
|
255
|
+
* `NEUROLINK_STT_PROVIDER` when it names a known provider or alias, else the
|
|
256
|
+
* first configured provider in descriptor order, else the built-in local
|
|
257
|
+
* engine. Never throws, never returns an empty string.
|
|
258
|
+
*/
|
|
259
|
+
export function resolveDefaultSTTProvider(credentials) {
|
|
260
|
+
const fromEnv = process.env.NEUROLINK_STT_PROVIDER?.trim();
|
|
261
|
+
if (fromEnv) {
|
|
262
|
+
const canonical = resolveSTTProviderName(fromEnv);
|
|
263
|
+
if (canonical) {
|
|
264
|
+
return canonical;
|
|
265
|
+
}
|
|
266
|
+
const endpoint = fromEnv.toLowerCase();
|
|
267
|
+
if (endpoint in resolveSTTEndpoints(credentials)) {
|
|
268
|
+
return endpoint;
|
|
269
|
+
}
|
|
270
|
+
logger.warn(`[STT] NEUROLINK_STT_PROVIDER="${fromEnv}" is not a known STT provider; ` +
|
|
271
|
+
`ignoring it. Known: ${STT_PROVIDER_DESCRIPTORS.map((d) => d.name).join(", ")}`);
|
|
272
|
+
}
|
|
273
|
+
const configured = STT_PROVIDER_DESCRIPTORS.find((d) => !d.capabilities.local && isSTTProviderConfigured(d.name, credentials));
|
|
274
|
+
return configured?.name ?? DEFAULT_LOCAL_STT_PROVIDER;
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* The variables that would configure each hosted STT provider, one clause per
|
|
278
|
+
* provider in precedence order, e.g. "OPENAI_STT_API_KEY or OPENAI_API_KEY for
|
|
279
|
+
* whisper, or DEEPGRAM_API_KEY for deepgram". Derived from the descriptors so
|
|
280
|
+
* a new provider appears in every "nothing is configured" message.
|
|
281
|
+
*/
|
|
282
|
+
export function describeSTTProviderKeys() {
|
|
283
|
+
return STT_PROVIDER_DESCRIPTORS.filter((d) => d.envVars.length > 0)
|
|
284
|
+
.map((d) => `${d.envVars.join(" or ")} for ${d.name}`)
|
|
285
|
+
.join(", or ");
|
|
286
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -70,7 +70,10 @@ export { AVATAR_ERROR_CODES, AvatarError } from "./utils/avatarProcessor.js";
|
|
|
70
70
|
export { MusicProcessor } from "./utils/musicProcessor.js";
|
|
71
71
|
export { MUSIC_ERROR_CODES, MusicError } from "./utils/musicProcessor.js";
|
|
72
72
|
export { STTProcessor } from "./utils/sttProcessor.js";
|
|
73
|
-
export {
|
|
73
|
+
export { describeSTTProviderKeys, isSTTProviderConfigured, resolveDefaultSTTProvider, STT_PROVIDER_DESCRIPTORS, } from "./factories/sttDescriptors.js";
|
|
74
|
+
export { buildSpeakerTurns, listSpeakers } from "./utils/speakerTurns.js";
|
|
75
|
+
export { correctTranscript } from "./voice/correction/index.js";
|
|
76
|
+
export { AzureTTS, AzureTTSHandler, CartesiaTTS, CartesiaTTSHandler, ElevenLabsTTS, ElevenLabsTTSHandler, FishAudioTTS, FishAudioTTSHandler, SixtyDBTTS, SixtyDBTTSHandler, GoogleTTSHandler, OpenAITTS, OpenAITTSHandler, AzureSTT, AzureSTTHandler, DeepgramSTT, DeepgramSTTHandler, ElevenLabsSTT, ElevenLabsSTTHandler, GoogleSTT, GoogleSTTHandler, OpenAISTT, OpenAISTTHandler, WhisperSTT, WhisperSTTHandler, WhistleSTT, WhistleSTTHandler, BaseRealtimeHandler, GeminiLive, GeminiLiveHandler, OpenAIRealtime, OpenAIRealtimeHandler, RealtimeProcessor, RealtimeError, STTError, VoiceError, registerDefaultRealtimeHandlers, registerDefaultSTTHandlers, registerDefaultTTSHandlers, } from "./voice/index.js";
|
|
74
77
|
export { BeatovenMusic, BeatovenMusicHandler, ElevenLabsMusic, ElevenLabsMusicHandler, LyriaMusic, LyriaMusicHandler, registerDefaultMusicHandlers, ReplicateMusic, ReplicateMusicHandler, } from "./music/index.js";
|
|
75
78
|
export { DIDAvatar, DIDAvatarHandler, HeyGenAvatar, HeyGenAvatarHandler, registerDefaultAvatarHandlers, ReplicateAvatar, ReplicateAvatarHandler, } from "./avatar/index.js";
|
|
76
79
|
export { isVertexVideoConfigured, KlingVideoHandler, registerDefaultVideoHandlers, ReplicateVideoHandler, RunwayVideoHandler, VertexVideoHandler, } from "./adapters/video/index.js";
|
package/dist/index.js
CHANGED
|
@@ -120,6 +120,12 @@ export { MusicProcessor } from "./utils/musicProcessor.js";
|
|
|
120
120
|
export { MUSIC_ERROR_CODES, MusicError } from "./utils/musicProcessor.js";
|
|
121
121
|
// STT / Realtime processors (registries for speech-to-text + live voice)
|
|
122
122
|
export { STTProcessor } from "./utils/sttProcessor.js";
|
|
123
|
+
// The `transcribe` inference type: STT provider descriptors (precedence
|
|
124
|
+
// order = default-provider order), provider-neutral speaker turns, and the
|
|
125
|
+
// transcript correction layer `neurolink.transcribe()` runs.
|
|
126
|
+
export { describeSTTProviderKeys, isSTTProviderConfigured, resolveDefaultSTTProvider, STT_PROVIDER_DESCRIPTORS, } from "./factories/sttDescriptors.js";
|
|
127
|
+
export { buildSpeakerTurns, listSpeakers } from "./utils/speakerTurns.js";
|
|
128
|
+
export { correctTranscript } from "./voice/correction/index.js";
|
|
123
129
|
// STT_ERROR_CODES is surfaced via the types barrel (export * from "./types/index.js")
|
|
124
130
|
// STTError is re-exported below from the voice/index.js barrel
|
|
125
131
|
// ============================================================================
|
|
@@ -134,7 +140,7 @@ export {
|
|
|
134
140
|
// TTS
|
|
135
141
|
AzureTTS, AzureTTSHandler, CartesiaTTS, CartesiaTTSHandler, ElevenLabsTTS, ElevenLabsTTSHandler, FishAudioTTS, FishAudioTTSHandler, SixtyDBTTS, SixtyDBTTSHandler, GoogleTTSHandler, OpenAITTS, OpenAITTSHandler,
|
|
136
142
|
// STT
|
|
137
|
-
AzureSTT, AzureSTTHandler, DeepgramSTT, DeepgramSTTHandler, ElevenLabsSTT, ElevenLabsSTTHandler, GoogleSTT, GoogleSTTHandler, OpenAISTT, OpenAISTTHandler, WhisperSTT, WhisperSTTHandler,
|
|
143
|
+
AzureSTT, AzureSTTHandler, DeepgramSTT, DeepgramSTTHandler, ElevenLabsSTT, ElevenLabsSTTHandler, GoogleSTT, GoogleSTTHandler, OpenAISTT, OpenAISTTHandler, WhisperSTT, WhisperSTTHandler, WhistleSTT, WhistleSTTHandler,
|
|
138
144
|
// Realtime
|
|
139
145
|
BaseRealtimeHandler, GeminiLive, GeminiLiveHandler, OpenAIRealtime, OpenAIRealtimeHandler, RealtimeProcessor,
|
|
140
146
|
// Voice error classes (RealtimeError + STTError + VoiceError)
|
package/dist/neurolink.d.ts
CHANGED
|
@@ -15,7 +15,7 @@ import type { DynamicOptions } from "./types/index.js";
|
|
|
15
15
|
import { SkillsManager } from "./skills/skillsManager.js";
|
|
16
16
|
import type { CostAnalysisOptions, CostAnalysisResult, ProviderMetricsOptions, ProviderMetricsResult, TeamAnalyticsOptions, TeamAnalyticsResult } from "./types/index.js";
|
|
17
17
|
import { TaskManager } from "./tasks/taskManager.js";
|
|
18
|
-
import type { SessionExport, SessionListItem } from "./types/index.js";
|
|
18
|
+
import type { TranscribeOptions, TranscribeResult, TranscribeStreamEvent, TranscribeStreamOptions, SessionExport, SessionListItem } from "./types/index.js";
|
|
19
19
|
/**
|
|
20
20
|
* Curator P2-4 dedup (concurrency-safe): native providers emit
|
|
21
21
|
* `generation:end` on the shared SDK emitter. We attach a fresh
|
|
@@ -2689,6 +2689,73 @@ export declare class NeuroLink {
|
|
|
2689
2689
|
* `decisionHooks` and emits `decision:before` / `decision:after`.
|
|
2690
2690
|
*/
|
|
2691
2691
|
tryDecide(options: DecisionCallerOptions): Promise<DecisionResult | null>;
|
|
2692
|
+
/**
|
|
2693
|
+
* Transcribe a recording to text through any STT provider, with the
|
|
2694
|
+
* optional layers that make self-hosted engines usable: a dictionary of
|
|
2695
|
+
* names and jargon (sent to the engine as context, then repaired after the
|
|
2696
|
+
* fact behind a `tryDecide` guard), an LLM rewrite, a second-opinion
|
|
2697
|
+
* engine, and a fallback engine for audio the primary cannot read.
|
|
2698
|
+
*
|
|
2699
|
+
* The provider defaults to `NEUROLINK_STT_PROVIDER`, then the first
|
|
2700
|
+
* configured STT provider, then the built-in local engine, so an instance
|
|
2701
|
+
* with no STT configuration still transcribes. Per-call
|
|
2702
|
+
* `credentials.stt` beats the instance's, which beats the environment.
|
|
2703
|
+
*
|
|
2704
|
+
* @example
|
|
2705
|
+
* ```typescript
|
|
2706
|
+
* const result = await neurolink.transcribe({
|
|
2707
|
+
* audio: "./meeting.wav",
|
|
2708
|
+
* dictionary: [{ term: "NeuroLink", heardAs: ["neural link"] }],
|
|
2709
|
+
* });
|
|
2710
|
+
* console.log(result.text, result.engine.provider, result.steps);
|
|
2711
|
+
* ```
|
|
2712
|
+
*
|
|
2713
|
+
* @throws STTError when the audio cannot be read, the provider is unknown,
|
|
2714
|
+
* or the engine fails with no fallback that covers errors
|
|
2715
|
+
*/
|
|
2716
|
+
transcribe(options: TranscribeOptions): Promise<TranscribeResult>;
|
|
2717
|
+
/**
|
|
2718
|
+
* Transcribe a live stream of PCM16LE frames. Engines with a native stream
|
|
2719
|
+
* use it; every other engine is streamed through the chunked adapter
|
|
2720
|
+
* (energy-gated utterances, rolling re-transcription, LocalAgreement
|
|
2721
|
+
* commits). Emits `interim`, `final`, `corrected` and the other
|
|
2722
|
+
* {@link TranscribeStreamEvent}s; the same correction, fallback and
|
|
2723
|
+
* provider rules as {@link NeuroLink.transcribe} apply per utterance.
|
|
2724
|
+
*
|
|
2725
|
+
* @example
|
|
2726
|
+
* ```typescript
|
|
2727
|
+
* for await (const event of neurolink.transcribeStream({ audio: micFrames })) {
|
|
2728
|
+
* if (event.type === "final") console.log(event.text);
|
|
2729
|
+
* }
|
|
2730
|
+
* ```
|
|
2731
|
+
*/
|
|
2732
|
+
transcribeStream(options: TranscribeStreamOptions): AsyncIterable<TranscribeStreamEvent>;
|
|
2733
|
+
/**
|
|
2734
|
+
* Instance `credentials.stt` with a per-call `stt` slice merged on top,
|
|
2735
|
+
* provider by provider and field by field (a per-call `apiKey` keeps the
|
|
2736
|
+
* instance's `baseURL`). Undefined when neither has one.
|
|
2737
|
+
*/
|
|
2738
|
+
private resolveSTTCredentials;
|
|
2739
|
+
/**
|
|
2740
|
+
* The STT provider for a call: an explicit name (or alias) as its
|
|
2741
|
+
* canonical name, a name only a caller-registered handler knows as is, and
|
|
2742
|
+
* no name as the default provider for these credentials.
|
|
2743
|
+
*/
|
|
2744
|
+
private resolveSTTProvider;
|
|
2745
|
+
/**
|
|
2746
|
+
* Everything the transcribe orchestration needs from this instance: STT
|
|
2747
|
+
* handlers with the request's credentials applied, provider resolution,
|
|
2748
|
+
* the fail-open decision model and a rewrite model.
|
|
2749
|
+
*/
|
|
2750
|
+
private buildTranscribeDeps;
|
|
2751
|
+
/**
|
|
2752
|
+
* The correction layer's two model calls: `decide` is {@link tryDecide}
|
|
2753
|
+
* (fail-open — `null` on any failure), `rewrite` a plain text call with
|
|
2754
|
+
* tools and memory off (`onPartial`, when given, receives the finished
|
|
2755
|
+
* text once — see the note inside on why it is not streamed) and aborted
|
|
2756
|
+
* at its timeout.
|
|
2757
|
+
*/
|
|
2758
|
+
private buildSTTCorrectionDeps;
|
|
2692
2759
|
/**
|
|
2693
2760
|
* `tryDecide` without the site handling: the try/catch around the call.
|
|
2694
2761
|
* Dispatches through `this.decide`, so a subclass override or an instance
|