@juspay/neurolink 12.48.2 → 12.49.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +616 -595
- package/dist/cli/commands/serve.d.ts +0 -3
- package/dist/cli/commands/serve.js +85 -3
- package/dist/cli/commands/transcribe.d.ts +24 -0
- package/dist/cli/commands/transcribe.js +718 -0
- package/dist/cli/factories/commandFactory.d.ts +10 -0
- package/dist/cli/factories/commandFactory.js +13 -0
- package/dist/cli/parser.js +3 -0
- package/dist/factories/mediaHandlerCatalog.js +11 -9
- package/dist/factories/providerDescriptors.js +5 -4
- package/dist/factories/sttDescriptors.d.ts +63 -0
- package/dist/factories/sttDescriptors.js +286 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +7 -1
- package/dist/neurolink.d.ts +68 -1
- package/dist/neurolink.js +202 -4
- package/dist/processors/media/AudioProcessor.d.ts +15 -56
- package/dist/processors/media/AudioProcessor.js +105 -276
- package/dist/processors/media/VideoProcessor.d.ts +3 -5
- package/dist/processors/media/VideoProcessor.js +52 -51
- package/dist/providers/cloudflareClef.js +9 -1
- package/dist/server/abstract/baseServerAdapter.d.ts +7 -0
- package/dist/server/abstract/baseServerAdapter.js +13 -1
- package/dist/server/adapters/expressAdapter.d.ts +1 -0
- package/dist/server/adapters/expressAdapter.js +48 -4
- package/dist/server/adapters/fastifyAdapter.d.ts +1 -0
- package/dist/server/adapters/fastifyAdapter.js +38 -1
- package/dist/server/adapters/honoAdapter.d.ts +1 -4
- package/dist/server/adapters/honoAdapter.js +83 -1
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +43 -0
- package/dist/server/index.d.ts +3 -2
- package/dist/server/index.js +3 -2
- package/dist/server/routes/index.d.ts +1 -0
- package/dist/server/routes/index.js +3 -0
- package/dist/server/routes/transcribeRoutes.d.ts +23 -0
- package/dist/server/routes/transcribeRoutes.js +506 -0
- package/dist/server/utils/rawBody.d.ts +32 -0
- package/dist/server/utils/rawBody.js +76 -0
- package/dist/server/utils/validation.d.ts +192 -0
- package/dist/server/utils/validation.js +100 -0
- package/dist/server/websocket/transcribeWebSocket.d.ts +42 -0
- package/dist/server/websocket/transcribeWebSocket.js +264 -0
- package/dist/types/cli.d.ts +46 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/providers.d.ts +7 -1
- package/dist/types/server.d.ts +117 -1
- package/dist/types/stt.d.ts +160 -7
- package/dist/types/transcribe.d.ts +506 -0
- package/dist/types/transcribe.js +29 -0
- package/dist/utils/speakerTurns.d.ts +28 -0
- package/dist/utils/speakerTurns.js +93 -0
- package/dist/utils/sttProcessor.d.ts +20 -2
- package/dist/utils/sttProcessor.js +73 -5
- package/dist/voice/correction/dictionary.d.ts +49 -0
- package/dist/voice/correction/dictionary.js +134 -0
- package/dist/voice/correction/guard.d.ts +35 -0
- package/dist/voice/correction/guard.js +161 -0
- package/dist/voice/correction/index.d.ts +47 -0
- package/dist/voice/correction/index.js +211 -0
- package/dist/voice/correction/rewrite.d.ts +92 -0
- package/dist/voice/correction/rewrite.js +322 -0
- package/dist/voice/errors.d.ts +15 -0
- package/dist/voice/errors.js +45 -0
- package/dist/voice/index.d.ts +20 -2
- package/dist/voice/index.js +75 -9
- package/dist/voice/providers/AzureSTT.d.ts +10 -7
- package/dist/voice/providers/AzureSTT.js +23 -75
- package/dist/voice/providers/DeepgramSTT.d.ts +17 -2
- package/dist/voice/providers/DeepgramSTT.js +109 -13
- package/dist/voice/providers/ElevenLabsSTT.d.ts +12 -2
- package/dist/voice/providers/ElevenLabsSTT.js +34 -8
- package/dist/voice/providers/GoogleSTT.d.ts +10 -8
- package/dist/voice/providers/GoogleSTT.js +21 -75
- package/dist/voice/providers/OpenAISTT.d.ts +30 -9
- package/dist/voice/providers/OpenAISTT.js +196 -21
- package/dist/voice/providers/WhistleSTT.d.ts +87 -0
- package/dist/voice/providers/WhistleSTT.js +313 -0
- package/dist/voice/streaming/chunkedStreamAdapter.d.ts +19 -0
- package/dist/voice/streaming/chunkedStreamAdapter.js +519 -0
- package/dist/voice/streaming/endpointer.d.ts +81 -0
- package/dist/voice/streaming/endpointer.js +247 -0
- package/dist/voice/streaming/localAgreement.d.ts +30 -0
- package/dist/voice/streaming/localAgreement.js +44 -0
- package/dist/voice/sttEnvCredentials.d.ts +12 -0
- package/dist/voice/sttEnvCredentials.js +12 -0
- package/dist/voice/transcribe.d.ts +47 -0
- package/dist/voice/transcribe.js +519 -0
- package/dist/voice/transcribeStream.d.ts +4 -0
- package/dist/voice/transcribeStream.js +584 -0
- package/dist/voice/whistle/assets.d.ts +47 -0
- package/dist/voice/whistle/assets.js +270 -0
- package/dist/voice/whistle/audio.d.ts +32 -0
- package/dist/voice/whistle/audio.js +267 -0
- package/dist/voice/whistle/engine.d.ts +32 -0
- package/dist/voice/whistle/engine.js +193 -0
- package/dist/voice/whistle/worker.d.ts +18 -0
- package/dist/voice/whistle/worker.js +294 -0
- package/docs-site/static/search-index.json +71 -12
- package/models/whistle/README.md +18 -0
- package/models/whistle/needle.js +2 -0
- package/models/whistle/needle.wasm +0 -0
- package/models/whistle/whistle.cact +0 -0
- package/package.json +7 -3
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module utils/sttProcessor
|
|
8
8
|
*/
|
|
9
|
-
import type { STTOptions, STTResult, STTHandler } from "../types/index.js";
|
|
9
|
+
import type { STTCredentials, STTOptions, STTResult, STTHandler } from "../types/index.js";
|
|
10
10
|
/**
|
|
11
11
|
* STT processor class for orchestrating speech-to-text operations
|
|
12
12
|
*
|
|
@@ -71,6 +71,19 @@ export declare class STTProcessor {
|
|
|
71
71
|
* @returns Handler instance or undefined if not registered
|
|
72
72
|
*/
|
|
73
73
|
static getHandler(providerName: string): STTHandler | undefined;
|
|
74
|
+
/**
|
|
75
|
+
* The handler a call should use. A credentials slice for a shipped
|
|
76
|
+
* provider builds a fresh handler for this call; otherwise the registered
|
|
77
|
+
* one is used, and a shipped provider that was never registered (its key
|
|
78
|
+
* arrived after registration, or it is the local engine on a registry that
|
|
79
|
+
* was cleared) is built from the environment rather than reported as
|
|
80
|
+
* unsupported. Names that are neither registered nor shipped resolve to
|
|
81
|
+
* `undefined`.
|
|
82
|
+
*
|
|
83
|
+
* The handler classes are loaded lazily: `voice/index.ts` imports this
|
|
84
|
+
* module, so a static import back would be a cycle.
|
|
85
|
+
*/
|
|
86
|
+
static resolveHandler(provider: string, credentials?: STTCredentials): Promise<STTHandler | undefined>;
|
|
74
87
|
/**
|
|
75
88
|
* List the names of all registered providers.
|
|
76
89
|
*/
|
|
@@ -107,6 +120,11 @@ export declare class STTProcessor {
|
|
|
107
120
|
* @param audio - Audio data as Buffer or ArrayBuffer
|
|
108
121
|
* @param provider - Provider identifier
|
|
109
122
|
* @param options - STT configuration options
|
|
123
|
+
* @param credentials - Per-call credentials (`credentials.stt`). When it
|
|
124
|
+
* carries a slice for this provider — even an empty one — a fresh handler
|
|
125
|
+
* is built from that slice (environment as fallback) for this call alone,
|
|
126
|
+
* instead of the registry's shared instance: per-call beats instance
|
|
127
|
+
* beats environment, exactly as for text providers.
|
|
110
128
|
* @returns Transcription result with text and metadata
|
|
111
129
|
* @throws STTError if validation fails or provider not supported/configured
|
|
112
130
|
*
|
|
@@ -121,5 +139,5 @@ export declare class STTProcessor {
|
|
|
121
139
|
* console.log(`Confidence: ${result.confidence}`);
|
|
122
140
|
* ```
|
|
123
141
|
*/
|
|
124
|
-
static transcribe(audio: Buffer | ArrayBuffer, provider: string, options: STTOptions): Promise<STTResult>;
|
|
142
|
+
static transcribe(audio: Buffer | ArrayBuffer, provider: string, options: STTOptions, credentials?: STTCredentials): Promise<STTResult>;
|
|
125
143
|
}
|
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
* @module utils/sttProcessor
|
|
8
8
|
*/
|
|
9
9
|
import { logger } from "./logger.js";
|
|
10
|
+
import { buildSpeakerTurns, listSpeakers } from "./speakerTurns.js";
|
|
11
|
+
import { getSTTProviderDescriptor, resolveCallSTTEndpoints, } from "../factories/sttDescriptors.js";
|
|
10
12
|
import { STT_ERROR_CODES } from "../types/index.js";
|
|
11
13
|
import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
|
|
12
14
|
import { STTError } from "../voice/errors.js";
|
|
@@ -84,6 +86,40 @@ export class STTProcessor {
|
|
|
84
86
|
static getHandler(providerName) {
|
|
85
87
|
return this.registry.get(providerName);
|
|
86
88
|
}
|
|
89
|
+
/**
|
|
90
|
+
* The handler a call should use. A credentials slice for a shipped
|
|
91
|
+
* provider builds a fresh handler for this call; otherwise the registered
|
|
92
|
+
* one is used, and a shipped provider that was never registered (its key
|
|
93
|
+
* arrived after registration, or it is the local engine on a registry that
|
|
94
|
+
* was cleared) is built from the environment rather than reported as
|
|
95
|
+
* unsupported. Names that are neither registered nor shipped resolve to
|
|
96
|
+
* `undefined`.
|
|
97
|
+
*
|
|
98
|
+
* The handler classes are loaded lazily: `voice/index.ts` imports this
|
|
99
|
+
* module, so a static import back would be a cycle.
|
|
100
|
+
*/
|
|
101
|
+
static async resolveHandler(provider, credentials) {
|
|
102
|
+
const endpoint = resolveCallSTTEndpoints(credentials)[provider.trim().toLowerCase()];
|
|
103
|
+
if (endpoint) {
|
|
104
|
+
// A per-call named endpoint: a fresh handler, never the registry's.
|
|
105
|
+
const { createSTTHandler } = await import("../voice/index.js");
|
|
106
|
+
return createSTTHandler(provider, credentials);
|
|
107
|
+
}
|
|
108
|
+
const descriptor = getSTTProviderDescriptor(provider);
|
|
109
|
+
const slice = descriptor
|
|
110
|
+
? credentials?.[descriptor.credentialsKey]
|
|
111
|
+
: undefined;
|
|
112
|
+
if (descriptor && slice !== undefined) {
|
|
113
|
+
const { createSTTHandler } = await import("../voice/index.js");
|
|
114
|
+
return createSTTHandler(descriptor.name, credentials);
|
|
115
|
+
}
|
|
116
|
+
const registered = this.getHandler(provider);
|
|
117
|
+
if (registered || !descriptor) {
|
|
118
|
+
return registered;
|
|
119
|
+
}
|
|
120
|
+
const { createSTTHandler } = await import("../voice/index.js");
|
|
121
|
+
return createSTTHandler(descriptor.name, credentials);
|
|
122
|
+
}
|
|
87
123
|
/**
|
|
88
124
|
* List the names of all registered providers.
|
|
89
125
|
*/
|
|
@@ -134,6 +170,11 @@ export class STTProcessor {
|
|
|
134
170
|
* @param audio - Audio data as Buffer or ArrayBuffer
|
|
135
171
|
* @param provider - Provider identifier
|
|
136
172
|
* @param options - STT configuration options
|
|
173
|
+
* @param credentials - Per-call credentials (`credentials.stt`). When it
|
|
174
|
+
* carries a slice for this provider — even an empty one — a fresh handler
|
|
175
|
+
* is built from that slice (environment as fallback) for this call alone,
|
|
176
|
+
* instead of the registry's shared instance: per-call beats instance
|
|
177
|
+
* beats environment, exactly as for text providers.
|
|
137
178
|
* @returns Transcription result with text and metadata
|
|
138
179
|
* @throws STTError if validation fails or provider not supported/configured
|
|
139
180
|
*
|
|
@@ -148,7 +189,7 @@ export class STTProcessor {
|
|
|
148
189
|
* console.log(`Confidence: ${result.confidence}`);
|
|
149
190
|
* ```
|
|
150
191
|
*/
|
|
151
|
-
static async transcribe(audio, provider, options) {
|
|
192
|
+
static async transcribe(audio, provider, options, credentials) {
|
|
152
193
|
// Create span early so preflight failures are captured
|
|
153
194
|
const span = SpanSerializer.createSpan(SpanType.STT, "stt.transcribe", {
|
|
154
195
|
"stt.operation": "transcribe",
|
|
@@ -186,7 +227,7 @@ export class STTProcessor {
|
|
|
186
227
|
});
|
|
187
228
|
}
|
|
188
229
|
// 2. Handler lookup and error if provider not supported
|
|
189
|
-
const handler = this.
|
|
230
|
+
const handler = await this.resolveHandler(provider, credentials);
|
|
190
231
|
if (!handler) {
|
|
191
232
|
logger.error(`[STTProcessor] Provider "${provider}" is not registered`);
|
|
192
233
|
throw new STTError({
|
|
@@ -230,9 +271,12 @@ export class STTProcessor {
|
|
|
230
271
|
// 4. Configuration check
|
|
231
272
|
if (!handler.isConfigured()) {
|
|
232
273
|
logger.warn(`[STTProcessor] Provider "${provider}" is not properly configured`);
|
|
274
|
+
const hint = handler.describeConfiguration?.();
|
|
233
275
|
throw new STTError({
|
|
234
276
|
code: STT_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
|
|
235
|
-
message:
|
|
277
|
+
message: hint
|
|
278
|
+
? `STT provider "${provider}" is not configured. ${hint}`
|
|
279
|
+
: `STT provider "${provider}" is not configured. Please set the required API keys.`,
|
|
236
280
|
category: ErrorCategory.CONFIGURATION,
|
|
237
281
|
severity: ErrorSeverity.HIGH,
|
|
238
282
|
retriable: false,
|
|
@@ -240,9 +284,20 @@ export class STTProcessor {
|
|
|
240
284
|
});
|
|
241
285
|
}
|
|
242
286
|
logger.debug(`[STTProcessor] Starting transcription with provider: ${provider}`);
|
|
287
|
+
// `diarization` is documented as an alias of `speakerDiarization`, but
|
|
288
|
+
// every handler reads only the latter; fold it in here so the alias
|
|
289
|
+
// works for all of them.
|
|
290
|
+
const handlerOptions = options.diarization && options.speakerDiarization === undefined
|
|
291
|
+
? { ...options, speakerDiarization: true }
|
|
292
|
+
: options;
|
|
243
293
|
// 5. Call handler.transcribe() - providers handle their own timeouts
|
|
244
|
-
const result = await handler.transcribe(audio,
|
|
245
|
-
// 6. Post-processing: enrich result with provider metadata
|
|
294
|
+
const result = await handler.transcribe(audio, handlerOptions);
|
|
295
|
+
// 6. Post-processing: enrich result with provider metadata, and give
|
|
296
|
+
// every diarizing provider the same shape. Providers label speakers in
|
|
297
|
+
// different places (per word, per segment); when a handler returned
|
|
298
|
+
// speaker-labelled words but no speaker-labelled segments, derive the
|
|
299
|
+
// turns here so callers can read `segments[].speaker` regardless of
|
|
300
|
+
// provider.
|
|
246
301
|
const enrichedResult = {
|
|
247
302
|
...result,
|
|
248
303
|
metadata: {
|
|
@@ -251,6 +306,19 @@ export class STTProcessor {
|
|
|
251
306
|
latency: result.metadata?.latency ?? 0,
|
|
252
307
|
},
|
|
253
308
|
};
|
|
309
|
+
const hasSpeakerSegments = enrichedResult.segments?.some((segment) => segment.speaker !== undefined);
|
|
310
|
+
if (!hasSpeakerSegments && enrichedResult.words) {
|
|
311
|
+
const turns = buildSpeakerTurns(enrichedResult.words);
|
|
312
|
+
if (turns.length > 0) {
|
|
313
|
+
enrichedResult.segments = turns;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (!enrichedResult.speakers?.length && enrichedResult.segments) {
|
|
317
|
+
const speakers = listSpeakers(enrichedResult.segments);
|
|
318
|
+
if (speakers.length > 0) {
|
|
319
|
+
enrichedResult.speakers = speakers;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
254
322
|
// Don't log transcript content at INFO — voice transcriptions can carry
|
|
255
323
|
// PII / health / financial data, and INFO is typically persisted in
|
|
256
324
|
// production log aggregation (CloudWatch, Datadog, etc.). GDPR / CCPA
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dictionary matching for transcript correction.
|
|
3
|
+
*
|
|
4
|
+
* A dictionary entry names a term (a person, product, acronym …), the ways
|
|
5
|
+
* speech engines mis-hear it, and what it means. Every alias found in a
|
|
6
|
+
* transcript becomes a *candidate*: it is not applied blindly, because the
|
|
7
|
+
* alias of a name is usually an ordinary word too ("part" for a name that
|
|
8
|
+
* sounds like it). The guard (`guard.ts`) decides per candidate; this module
|
|
9
|
+
* only finds them and applies the verdicts.
|
|
10
|
+
*
|
|
11
|
+
* Matching is whole-word and script-agnostic: a word boundary is any
|
|
12
|
+
* character that is not a letter, a combining mark or a digit. Marks matter
|
|
13
|
+
* for Indic scripts, where vowel signs and viramas are `\p{M}`, not `\p{L}` —
|
|
14
|
+
* a letters-only boundary would let an alias match the first half of a word.
|
|
15
|
+
*
|
|
16
|
+
* @module voice/correction/dictionary
|
|
17
|
+
*/
|
|
18
|
+
import type { STTDecisionRecord, STTDictionaryCandidate, STTDictionaryEntry } from "../../types/index.js";
|
|
19
|
+
/** Whether `text` contains any Indic-script character. */
|
|
20
|
+
export declare function hasIndicScript(text: string): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Every alias of every entry that occurs in `text` as a whole word.
|
|
23
|
+
*
|
|
24
|
+
* An entry whose term already appears in the text contributes no candidates:
|
|
25
|
+
* the engine heard the term itself, so an alias elsewhere in the same text is
|
|
26
|
+
* more likely the ordinary word. An alias in an Indic script is skipped when
|
|
27
|
+
* the text has none (it cannot match, and the check is cheaper than a regex).
|
|
28
|
+
* Each (entry, alias) pair is reported once, however often it occurs.
|
|
29
|
+
*/
|
|
30
|
+
export declare function findDictionaryCandidates(text: string, dictionary: readonly STTDictionaryEntry[] | undefined): STTDictionaryCandidate[];
|
|
31
|
+
/**
|
|
32
|
+
* Replace every candidate the guard did not call `"literal"` with its term.
|
|
33
|
+
*
|
|
34
|
+
* `decisions[i]` belongs to `candidates[i]`; a missing decision applies the
|
|
35
|
+
* term, which is the same fail-open default the guard uses. Every occurrence
|
|
36
|
+
* is replaced — the decision was asked about the alias in this transcript, so
|
|
37
|
+
* all of its occurrences share the answer.
|
|
38
|
+
*/
|
|
39
|
+
export declare function applyDictionary(text: string, candidates: readonly STTDictionaryCandidate[], decisions: readonly STTDecisionRecord[]): string;
|
|
40
|
+
/**
|
|
41
|
+
* The dictionary as a table for the rewrite model's system prompt, or `""`
|
|
42
|
+
* when there is nothing to show.
|
|
43
|
+
*/
|
|
44
|
+
export declare function dictionaryBlock(dictionary: readonly STTDictionaryEntry[] | undefined): string;
|
|
45
|
+
/**
|
|
46
|
+
* Distinct dictionary terms, in dictionary order, for engines that bias
|
|
47
|
+
* recognition on a vocabulary or a context prompt.
|
|
48
|
+
*/
|
|
49
|
+
export declare function dictionaryVocabulary(dictionary: readonly STTDictionaryEntry[] | undefined): string[];
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dictionary matching for transcript correction.
|
|
3
|
+
*
|
|
4
|
+
* A dictionary entry names a term (a person, product, acronym …), the ways
|
|
5
|
+
* speech engines mis-hear it, and what it means. Every alias found in a
|
|
6
|
+
* transcript becomes a *candidate*: it is not applied blindly, because the
|
|
7
|
+
* alias of a name is usually an ordinary word too ("part" for a name that
|
|
8
|
+
* sounds like it). The guard (`guard.ts`) decides per candidate; this module
|
|
9
|
+
* only finds them and applies the verdicts.
|
|
10
|
+
*
|
|
11
|
+
* Matching is whole-word and script-agnostic: a word boundary is any
|
|
12
|
+
* character that is not a letter, a combining mark or a digit. Marks matter
|
|
13
|
+
* for Indic scripts, where vowel signs and viramas are `\p{M}`, not `\p{L}` —
|
|
14
|
+
* a letters-only boundary would let an alias match the first half of a word.
|
|
15
|
+
*
|
|
16
|
+
* @module voice/correction/dictionary
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Any Indic script, Devanagari (U+0900) through Sinhala (U+0DFF): Bengali,
|
|
20
|
+
* Gurmukhi, Gujarati, Odia, Tamil, Telugu, Kannada and Malayalam sit between.
|
|
21
|
+
*/
|
|
22
|
+
const INDIC_SCRIPT_RE = /[ऀ-]/u;
|
|
23
|
+
/** Characters that continue a word: letters, combining marks, digits. */
|
|
24
|
+
const WORD_CHAR = "[\\p{L}\\p{M}\\p{N}]";
|
|
25
|
+
function escapeRegExp(text) {
|
|
26
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Whole-word, case-insensitive matcher for `phrase`. Whitespace inside a
|
|
30
|
+
* multi-word alias matches any run of whitespace, since engines differ in how
|
|
31
|
+
* they space words.
|
|
32
|
+
*/
|
|
33
|
+
function wholeWordPattern(phrase, flags = "iu") {
|
|
34
|
+
const body = escapeRegExp(phrase.trim()).replace(/\s+/g, "\\s+");
|
|
35
|
+
return new RegExp(`(?<!${WORD_CHAR})${body}(?!${WORD_CHAR})`, flags);
|
|
36
|
+
}
|
|
37
|
+
/** Whether `text` contains any Indic-script character. */
|
|
38
|
+
export function hasIndicScript(text) {
|
|
39
|
+
return INDIC_SCRIPT_RE.test(text);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Every alias of every entry that occurs in `text` as a whole word.
|
|
43
|
+
*
|
|
44
|
+
* An entry whose term already appears in the text contributes no candidates:
|
|
45
|
+
* the engine heard the term itself, so an alias elsewhere in the same text is
|
|
46
|
+
* more likely the ordinary word. An alias in an Indic script is skipped when
|
|
47
|
+
* the text has none (it cannot match, and the check is cheaper than a regex).
|
|
48
|
+
* Each (entry, alias) pair is reported once, however often it occurs.
|
|
49
|
+
*/
|
|
50
|
+
export function findDictionaryCandidates(text, dictionary) {
|
|
51
|
+
if (!text || !dictionary?.length) {
|
|
52
|
+
return [];
|
|
53
|
+
}
|
|
54
|
+
const textHasIndic = hasIndicScript(text);
|
|
55
|
+
const out = [];
|
|
56
|
+
for (const entry of dictionary) {
|
|
57
|
+
const term = entry?.term?.trim();
|
|
58
|
+
if (!term || !entry.heardAs?.length) {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (wholeWordPattern(term).test(text)) {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const seen = new Set();
|
|
65
|
+
for (const alias of entry.heardAs) {
|
|
66
|
+
const heard = typeof alias === "string" ? alias.trim() : "";
|
|
67
|
+
if (!heard || seen.has(heard.toLowerCase())) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
seen.add(heard.toLowerCase());
|
|
71
|
+
if (hasIndicScript(heard) && !textHasIndic) {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (wholeWordPattern(heard).test(text)) {
|
|
75
|
+
out.push({ entry, heard });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Replace every candidate the guard did not call `"literal"` with its term.
|
|
83
|
+
*
|
|
84
|
+
* `decisions[i]` belongs to `candidates[i]`; a missing decision applies the
|
|
85
|
+
* term, which is the same fail-open default the guard uses. Every occurrence
|
|
86
|
+
* is replaced — the decision was asked about the alias in this transcript, so
|
|
87
|
+
* all of its occurrences share the answer.
|
|
88
|
+
*/
|
|
89
|
+
export function applyDictionary(text, candidates, decisions) {
|
|
90
|
+
let out = text;
|
|
91
|
+
candidates.forEach((candidate, index) => {
|
|
92
|
+
if (decisions[index]?.choice === "literal") {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
const term = candidate.entry.term.trim();
|
|
96
|
+
// A function replacement, so a "$" in the term is never read as a group reference.
|
|
97
|
+
out = out.replace(wholeWordPattern(candidate.heard, "giu"), () => term);
|
|
98
|
+
});
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* The dictionary as a table for the rewrite model's system prompt, or `""`
|
|
103
|
+
* when there is nothing to show.
|
|
104
|
+
*/
|
|
105
|
+
export function dictionaryBlock(dictionary) {
|
|
106
|
+
const rows = (dictionary ?? [])
|
|
107
|
+
.filter((entry) => entry?.term?.trim())
|
|
108
|
+
.map((entry) => {
|
|
109
|
+
const heard = (entry.heardAs ?? [])
|
|
110
|
+
.map((alias) => alias.trim())
|
|
111
|
+
.filter(Boolean)
|
|
112
|
+
.join(", ");
|
|
113
|
+
return `${entry.term.trim()} | ${heard || "-"} | ${entry.meaning?.trim() || "-"}`;
|
|
114
|
+
});
|
|
115
|
+
return rows.length
|
|
116
|
+
? `Dictionary (term | also heard as | meaning):\n${rows.join("\n")}`
|
|
117
|
+
: "";
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Distinct dictionary terms, in dictionary order, for engines that bias
|
|
121
|
+
* recognition on a vocabulary or a context prompt.
|
|
122
|
+
*/
|
|
123
|
+
export function dictionaryVocabulary(dictionary) {
|
|
124
|
+
const seen = new Set();
|
|
125
|
+
const out = [];
|
|
126
|
+
for (const entry of dictionary ?? []) {
|
|
127
|
+
const term = entry?.term?.trim();
|
|
128
|
+
if (term && !seen.has(term.toLowerCase())) {
|
|
129
|
+
seen.add(term.toLowerCase());
|
|
130
|
+
out.push(term);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decision-model guard for dictionary substitutions.
|
|
3
|
+
*
|
|
4
|
+
* An alias is usually an ordinary word as well ("part" is how engines hear a
|
|
5
|
+
* name that sounds like it), so applying the dictionary blindly rewrites the
|
|
6
|
+
* literal word every time it is said. The guard asks a decision model, in one
|
|
7
|
+
* request, one question per candidate: did the speaker mean the term, or the
|
|
8
|
+
* word as written?
|
|
9
|
+
*
|
|
10
|
+
* Why a `choice` and not a yes/no: a boolean "is this the name X?" question
|
|
11
|
+
* was measured to be leading — it said yes to nearly everything. A choice
|
|
12
|
+
* between the term and the literal reading, each with its own rubric,
|
|
13
|
+
* discriminates: "my name is part" → the term at 97%, "part of the plan" →
|
|
14
|
+
* literal. Answers are matched by position (`candidate__N`), never by the
|
|
15
|
+
* alias text, so an alias the wire normalises cannot misroute an answer.
|
|
16
|
+
*
|
|
17
|
+
* Fail-open, like every other `decide` consumer: with no decision provider,
|
|
18
|
+
* on timeout (default 4 s) or on any error the dictionary is applied as is,
|
|
19
|
+
* and each record says why in `note`.
|
|
20
|
+
*
|
|
21
|
+
* @module voice/correction/guard
|
|
22
|
+
*/
|
|
23
|
+
import type { STTCorrectionDeps, STTDecisionRecord, STTDictionaryCandidate } from "../../types/index.js";
|
|
24
|
+
/** Default guard timeout; on expiry the dictionary is applied as is. */
|
|
25
|
+
export declare const DEFAULT_DECIDE_TIMEOUT_MS = 4000;
|
|
26
|
+
/**
|
|
27
|
+
* Judge every candidate in one decision request. Returns one record per
|
|
28
|
+
* candidate, in candidate order; a candidate the guard could not judge is
|
|
29
|
+
* recorded as `"term"` with a `note` saying why, so `applyDictionary` applies
|
|
30
|
+
* it exactly as it would without a guard.
|
|
31
|
+
*/
|
|
32
|
+
export declare function decideCandidates(text: string, candidates: readonly STTDictionaryCandidate[], decide: STTCorrectionDeps["decide"] | undefined, options?: {
|
|
33
|
+
context?: string;
|
|
34
|
+
timeoutMs?: number;
|
|
35
|
+
}): Promise<STTDecisionRecord[]>;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decision-model guard for dictionary substitutions.
|
|
3
|
+
*
|
|
4
|
+
* An alias is usually an ordinary word as well ("part" is how engines hear a
|
|
5
|
+
* name that sounds like it), so applying the dictionary blindly rewrites the
|
|
6
|
+
* literal word every time it is said. The guard asks a decision model, in one
|
|
7
|
+
* request, one question per candidate: did the speaker mean the term, or the
|
|
8
|
+
* word as written?
|
|
9
|
+
*
|
|
10
|
+
* Why a `choice` and not a yes/no: a boolean "is this the name X?" question
|
|
11
|
+
* was measured to be leading — it said yes to nearly everything. A choice
|
|
12
|
+
* between the term and the literal reading, each with its own rubric,
|
|
13
|
+
* discriminates: "my name is part" → the term at 97%, "part of the plan" →
|
|
14
|
+
* literal. Answers are matched by position (`candidate__N`), never by the
|
|
15
|
+
* alias text, so an alias the wire normalises cannot misroute an answer.
|
|
16
|
+
*
|
|
17
|
+
* Fail-open, like every other `decide` consumer: with no decision provider,
|
|
18
|
+
* on timeout (default 4 s) or on any error the dictionary is applied as is,
|
|
19
|
+
* and each record says why in `note`.
|
|
20
|
+
*
|
|
21
|
+
* @module voice/correction/guard
|
|
22
|
+
*/
|
|
23
|
+
import { decisionKey } from "../../utils/decisionAnswers.js";
|
|
24
|
+
import { TimeoutError, withTimeout } from "../../utils/async/withTimeout.js";
|
|
25
|
+
import { logger } from "../../utils/logger.js";
|
|
26
|
+
/** Namespace for the per-candidate questions. */
|
|
27
|
+
const CANDIDATE_NAMESPACE = "candidate";
|
|
28
|
+
/** Default guard timeout; on expiry the dictionary is applied as is. */
|
|
29
|
+
export const DEFAULT_DECIDE_TIMEOUT_MS = 4000;
|
|
30
|
+
/** Sent as `context` when the caller gave none. */
|
|
31
|
+
const DEFAULT_CONTEXT = "Speech transcript; names and product terms are often misheard.";
|
|
32
|
+
/**
|
|
33
|
+
* Upper bound on the transcript sent as state. Decision providers declare
|
|
34
|
+
* state windows (some well under 1,000 tokens); an over-limit request is
|
|
35
|
+
* refused before any network call and the guard fails open, so a long
|
|
36
|
+
* transcript costs nothing but the decision — still, the words around a
|
|
37
|
+
* candidate carry the signal, and a bounded state keeps the request cheap.
|
|
38
|
+
*/
|
|
39
|
+
const MAX_STATE_CHARS = 4000;
|
|
40
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
41
|
+
const finiteNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
42
|
+
function candidateQuestion(candidate) {
|
|
43
|
+
const { entry, heard } = candidate;
|
|
44
|
+
const meaning = entry.meaning?.trim();
|
|
45
|
+
return {
|
|
46
|
+
type: "choice",
|
|
47
|
+
instructions: `The speech recognizer wrote "${heard}". Which did the speaker mean?`,
|
|
48
|
+
criteria: {
|
|
49
|
+
term: `the term "${entry.term.trim()}"${meaning ? `: ${meaning}` : ""}`,
|
|
50
|
+
literal: `"${heard}" as an ordinary word or other name, meant literally`,
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Read one choice answer without trusting its shape: the deps type hands the
|
|
56
|
+
* answer map over as `unknown`, and a provider may omit an id or answer with
|
|
57
|
+
* a different type. Returns undefined when there is no usable term/literal
|
|
58
|
+
* verdict.
|
|
59
|
+
*/
|
|
60
|
+
function readTermOrLiteral(answer) {
|
|
61
|
+
if (!isRecord(answer) ||
|
|
62
|
+
(answer.type !== undefined && answer.type !== "choice")) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
const probabilities = isRecord(answer.probabilities)
|
|
66
|
+
? answer.probabilities
|
|
67
|
+
: {};
|
|
68
|
+
const pTerm = finiteNumber(probabilities.term);
|
|
69
|
+
const pLiteral = finiteNumber(probabilities.literal);
|
|
70
|
+
let choice;
|
|
71
|
+
if (answer.choice === "term" || answer.choice === "literal") {
|
|
72
|
+
choice = answer.choice;
|
|
73
|
+
}
|
|
74
|
+
else if (pTerm !== undefined || pLiteral !== undefined) {
|
|
75
|
+
// No winner named, but a distribution: take its argmax.
|
|
76
|
+
const term = pTerm ?? 1 - (pLiteral ?? 0);
|
|
77
|
+
const literal = pLiteral ?? 1 - term;
|
|
78
|
+
choice = term >= literal ? "term" : "literal";
|
|
79
|
+
}
|
|
80
|
+
if (!choice) {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
const probability = choice === "term"
|
|
84
|
+
? (pTerm ?? (pLiteral !== undefined ? 1 - pLiteral : undefined))
|
|
85
|
+
: (pLiteral ?? (pTerm !== undefined ? 1 - pTerm : undefined));
|
|
86
|
+
return { choice, probability, confidence: finiteNumber(answer.confidence) };
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Judge every candidate in one decision request. Returns one record per
|
|
90
|
+
* candidate, in candidate order; a candidate the guard could not judge is
|
|
91
|
+
* recorded as `"term"` with a `note` saying why, so `applyDictionary` applies
|
|
92
|
+
* it exactly as it would without a guard.
|
|
93
|
+
*/
|
|
94
|
+
export async function decideCandidates(text, candidates, decide, options) {
|
|
95
|
+
if (candidates.length === 0) {
|
|
96
|
+
return [];
|
|
97
|
+
}
|
|
98
|
+
const asIs = (note) => candidates.map((candidate) => ({
|
|
99
|
+
heard: candidate.heard,
|
|
100
|
+
term: candidate.entry.term.trim(),
|
|
101
|
+
choice: "term",
|
|
102
|
+
note,
|
|
103
|
+
}));
|
|
104
|
+
if (typeof decide !== "function") {
|
|
105
|
+
return asIs("no decision provider");
|
|
106
|
+
}
|
|
107
|
+
const questions = {};
|
|
108
|
+
candidates.forEach((candidate, index) => {
|
|
109
|
+
questions[decisionKey(CANDIDATE_NAMESPACE, index)] =
|
|
110
|
+
candidateQuestion(candidate);
|
|
111
|
+
});
|
|
112
|
+
const timeoutMs = options?.timeoutMs && options.timeoutMs > 0
|
|
113
|
+
? options.timeoutMs
|
|
114
|
+
: DEFAULT_DECIDE_TIMEOUT_MS;
|
|
115
|
+
let result;
|
|
116
|
+
try {
|
|
117
|
+
// The provider gets the timeout too, but the race is what guarantees it:
|
|
118
|
+
// a decide implementation that ignores `timeoutMs` must not stall the
|
|
119
|
+
// transcript.
|
|
120
|
+
result = await withTimeout(decide({
|
|
121
|
+
state: {
|
|
122
|
+
transcript: text.slice(0, MAX_STATE_CHARS),
|
|
123
|
+
context: options?.context?.trim() || DEFAULT_CONTEXT,
|
|
124
|
+
},
|
|
125
|
+
questions,
|
|
126
|
+
timeoutMs,
|
|
127
|
+
}), timeoutMs);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
const timedOut = error instanceof TimeoutError;
|
|
131
|
+
logger.debug("[transcribe] decision guard failed open", {
|
|
132
|
+
candidates: candidates.length,
|
|
133
|
+
timedOut,
|
|
134
|
+
});
|
|
135
|
+
return asIs(timedOut ? "decision timed out" : "decision unavailable");
|
|
136
|
+
}
|
|
137
|
+
// `tryDecide` returns null both when no decision provider is configured and
|
|
138
|
+
// when the call failed; the two are indistinguishable from here.
|
|
139
|
+
if (!result || !isRecord(result.answers)) {
|
|
140
|
+
return asIs("decision unavailable");
|
|
141
|
+
}
|
|
142
|
+
const answers = result.answers;
|
|
143
|
+
return candidates.map((candidate, index) => {
|
|
144
|
+
const verdict = readTermOrLiteral(answers[decisionKey(CANDIDATE_NAMESPACE, index)]);
|
|
145
|
+
const record = {
|
|
146
|
+
heard: candidate.heard,
|
|
147
|
+
term: candidate.entry.term.trim(),
|
|
148
|
+
choice: verdict?.choice ?? "term",
|
|
149
|
+
};
|
|
150
|
+
if (verdict?.probability !== undefined) {
|
|
151
|
+
record.probability = verdict.probability;
|
|
152
|
+
}
|
|
153
|
+
if (verdict?.confidence !== undefined) {
|
|
154
|
+
record.confidence = verdict.confidence;
|
|
155
|
+
}
|
|
156
|
+
if (!verdict) {
|
|
157
|
+
record.note = "no answer · applied as is";
|
|
158
|
+
}
|
|
159
|
+
return record;
|
|
160
|
+
});
|
|
161
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcript correction: dictionary candidates → decision-model guard → LLM
|
|
3
|
+
* rewrite.
|
|
4
|
+
*
|
|
5
|
+
* Every layer fails open and none may lose words. With no decision provider,
|
|
6
|
+
* on a guard timeout or error, the dictionary is applied as is; on a rewrite
|
|
7
|
+
* timeout or error, or a rewrite that drops too many words, the
|
|
8
|
+
* dictionary-applied text is kept. The result's `steps` say which of those
|
|
9
|
+
* happened, so a silently degraded path is still visible to the caller.
|
|
10
|
+
*
|
|
11
|
+
* @module voice/correction
|
|
12
|
+
*/
|
|
13
|
+
import { applyDictionary, dictionaryBlock, dictionaryVocabulary, findDictionaryCandidates } from "./dictionary.js";
|
|
14
|
+
import { decideCandidates } from "./guard.js";
|
|
15
|
+
import { buildRewriteSystemPrompt, buildRewriteUserPrompt, rewriteLines, rewriteTranscript } from "./rewrite.js";
|
|
16
|
+
import type { STTCorrectionDeps, STTCorrectionInput, STTCorrectionOptions, STTCorrectionOutput, STTDecisionRecord, STTDictionaryEntry } from "../../types/index.js";
|
|
17
|
+
export { applyDictionary, buildRewriteSystemPrompt, buildRewriteUserPrompt, decideCandidates, dictionaryBlock, dictionaryVocabulary, findDictionaryCandidates, rewriteLines, rewriteTranscript, };
|
|
18
|
+
/** Correction is on when asked for, or by default when a dictionary or a rewrite is configured. */
|
|
19
|
+
export declare function isCorrectionEnabled(dictionary: readonly STTDictionaryEntry[] | undefined, options: STTCorrectionOptions | undefined): boolean;
|
|
20
|
+
/**
|
|
21
|
+
* Correct one transcript (a whole short recording, or one final utterance
|
|
22
|
+
* when streaming). Never throws on a layer's failure and never returns empty
|
|
23
|
+
* text when either transcript had words.
|
|
24
|
+
*
|
|
25
|
+
* When the primary engine reported a low language score
|
|
26
|
+
* (`languageDetected === false`) and a second opinion exists, the second
|
|
27
|
+
* opinion leads: an Indic engine on English speech produces confident-looking
|
|
28
|
+
* nonsense, and the English-oriented engine's text is the better base. The
|
|
29
|
+
* engine text then goes to the rewrite as the secondary transcript, so
|
|
30
|
+
* Indian-language words it did catch are not lost.
|
|
31
|
+
*/
|
|
32
|
+
export declare function correctTranscript(input: STTCorrectionInput, deps: STTCorrectionDeps): Promise<STTCorrectionOutput>;
|
|
33
|
+
/**
|
|
34
|
+
* Correct a long transcript line by line (file transcription, diarized
|
|
35
|
+
* turns). With the rewrite on, lines go to the model in numbered batches of
|
|
36
|
+
* 25 and the dictionary travels in the prompt — the model sees each alias in
|
|
37
|
+
* its sentence, which is the context the guard would otherwise supply. With
|
|
38
|
+
* `rewrite: false`, each line with candidates gets its own guard call (up to
|
|
39
|
+
* four at a time) and the dictionary is applied per line. The result always
|
|
40
|
+
* has exactly as many lines as the input.
|
|
41
|
+
*/
|
|
42
|
+
export declare function correctTranscriptLines(lines: readonly string[], dictionary: readonly STTDictionaryEntry[] | undefined, options: STTCorrectionOptions | undefined, deps: STTCorrectionDeps): Promise<{
|
|
43
|
+
lines: string[];
|
|
44
|
+
decisions: STTDecisionRecord[];
|
|
45
|
+
steps: string[];
|
|
46
|
+
timings: STTCorrectionOutput["timings"];
|
|
47
|
+
}>;
|