@juspay/neurolink 12.48.2 → 12.49.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +616 -595
- package/dist/cli/commands/serve.d.ts +0 -3
- package/dist/cli/commands/serve.js +85 -3
- package/dist/cli/commands/transcribe.d.ts +24 -0
- package/dist/cli/commands/transcribe.js +718 -0
- package/dist/cli/factories/commandFactory.d.ts +10 -0
- package/dist/cli/factories/commandFactory.js +13 -0
- package/dist/cli/parser.js +3 -0
- package/dist/factories/mediaHandlerCatalog.js +11 -9
- package/dist/factories/providerDescriptors.js +5 -4
- package/dist/factories/sttDescriptors.d.ts +63 -0
- package/dist/factories/sttDescriptors.js +286 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +7 -1
- package/dist/neurolink.d.ts +68 -1
- package/dist/neurolink.js +202 -4
- package/dist/processors/media/AudioProcessor.d.ts +15 -56
- package/dist/processors/media/AudioProcessor.js +105 -276
- package/dist/processors/media/VideoProcessor.d.ts +3 -5
- package/dist/processors/media/VideoProcessor.js +52 -51
- package/dist/providers/cloudflareClef.js +9 -1
- package/dist/server/abstract/baseServerAdapter.d.ts +7 -0
- package/dist/server/abstract/baseServerAdapter.js +13 -1
- package/dist/server/adapters/expressAdapter.d.ts +1 -0
- package/dist/server/adapters/expressAdapter.js +48 -4
- package/dist/server/adapters/fastifyAdapter.d.ts +1 -0
- package/dist/server/adapters/fastifyAdapter.js +38 -1
- package/dist/server/adapters/honoAdapter.d.ts +1 -4
- package/dist/server/adapters/honoAdapter.js +83 -1
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +43 -0
- package/dist/server/index.d.ts +3 -2
- package/dist/server/index.js +3 -2
- package/dist/server/routes/index.d.ts +1 -0
- package/dist/server/routes/index.js +3 -0
- package/dist/server/routes/transcribeRoutes.d.ts +23 -0
- package/dist/server/routes/transcribeRoutes.js +506 -0
- package/dist/server/utils/rawBody.d.ts +32 -0
- package/dist/server/utils/rawBody.js +76 -0
- package/dist/server/utils/validation.d.ts +192 -0
- package/dist/server/utils/validation.js +100 -0
- package/dist/server/websocket/transcribeWebSocket.d.ts +42 -0
- package/dist/server/websocket/transcribeWebSocket.js +264 -0
- package/dist/types/cli.d.ts +46 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/providers.d.ts +7 -1
- package/dist/types/server.d.ts +117 -1
- package/dist/types/stt.d.ts +160 -7
- package/dist/types/transcribe.d.ts +506 -0
- package/dist/types/transcribe.js +29 -0
- package/dist/utils/speakerTurns.d.ts +28 -0
- package/dist/utils/speakerTurns.js +93 -0
- package/dist/utils/sttProcessor.d.ts +20 -2
- package/dist/utils/sttProcessor.js +73 -5
- package/dist/voice/correction/dictionary.d.ts +49 -0
- package/dist/voice/correction/dictionary.js +134 -0
- package/dist/voice/correction/guard.d.ts +35 -0
- package/dist/voice/correction/guard.js +161 -0
- package/dist/voice/correction/index.d.ts +47 -0
- package/dist/voice/correction/index.js +211 -0
- package/dist/voice/correction/rewrite.d.ts +92 -0
- package/dist/voice/correction/rewrite.js +322 -0
- package/dist/voice/errors.d.ts +15 -0
- package/dist/voice/errors.js +45 -0
- package/dist/voice/index.d.ts +20 -2
- package/dist/voice/index.js +75 -9
- package/dist/voice/providers/AzureSTT.d.ts +10 -7
- package/dist/voice/providers/AzureSTT.js +23 -75
- package/dist/voice/providers/DeepgramSTT.d.ts +17 -2
- package/dist/voice/providers/DeepgramSTT.js +109 -13
- package/dist/voice/providers/ElevenLabsSTT.d.ts +12 -2
- package/dist/voice/providers/ElevenLabsSTT.js +34 -8
- package/dist/voice/providers/GoogleSTT.d.ts +10 -8
- package/dist/voice/providers/GoogleSTT.js +21 -75
- package/dist/voice/providers/OpenAISTT.d.ts +30 -9
- package/dist/voice/providers/OpenAISTT.js +196 -21
- package/dist/voice/providers/WhistleSTT.d.ts +87 -0
- package/dist/voice/providers/WhistleSTT.js +313 -0
- package/dist/voice/streaming/chunkedStreamAdapter.d.ts +19 -0
- package/dist/voice/streaming/chunkedStreamAdapter.js +519 -0
- package/dist/voice/streaming/endpointer.d.ts +81 -0
- package/dist/voice/streaming/endpointer.js +247 -0
- package/dist/voice/streaming/localAgreement.d.ts +30 -0
- package/dist/voice/streaming/localAgreement.js +44 -0
- package/dist/voice/sttEnvCredentials.d.ts +12 -0
- package/dist/voice/sttEnvCredentials.js +12 -0
- package/dist/voice/transcribe.d.ts +47 -0
- package/dist/voice/transcribe.js +519 -0
- package/dist/voice/transcribeStream.d.ts +4 -0
- package/dist/voice/transcribeStream.js +584 -0
- package/dist/voice/whistle/assets.d.ts +47 -0
- package/dist/voice/whistle/assets.js +270 -0
- package/dist/voice/whistle/audio.d.ts +32 -0
- package/dist/voice/whistle/audio.js +267 -0
- package/dist/voice/whistle/engine.d.ts +32 -0
- package/dist/voice/whistle/engine.js +193 -0
- package/dist/voice/whistle/worker.d.ts +18 -0
- package/dist/voice/whistle/worker.js +294 -0
- package/docs-site/static/search-index.json +71 -12
- package/models/whistle/README.md +18 -0
- package/models/whistle/needle.js +2 -0
- package/models/whistle/needle.wasm +0 -0
- package/models/whistle/whistle.cact +0 -0
- package/package.json +7 -3
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcript correction: dictionary candidates → decision-model guard → LLM
|
|
3
|
+
* rewrite.
|
|
4
|
+
*
|
|
5
|
+
* Every layer fails open and none may lose words. With no decision provider,
|
|
6
|
+
* on a guard timeout or error, the dictionary is applied as is; on a rewrite
|
|
7
|
+
* timeout or error, or a rewrite that drops too many words, the
|
|
8
|
+
* dictionary-applied text is kept. The result's `steps` say which of those
|
|
9
|
+
* happened, so a silently degraded path is still visible to the caller.
|
|
10
|
+
*
|
|
11
|
+
* @module voice/correction
|
|
12
|
+
*/
|
|
13
|
+
import { applyDictionary, dictionaryBlock, dictionaryVocabulary, findDictionaryCandidates, } from "./dictionary.js";
|
|
14
|
+
import { DEFAULT_DECIDE_TIMEOUT_MS, decideCandidates } from "./guard.js";
|
|
15
|
+
import { buildRewriteSystemPrompt, buildRewriteUserPrompt, rewriteLines, rewriteTranscript, } from "./rewrite.js";
|
|
16
|
+
export { applyDictionary, buildRewriteSystemPrompt, buildRewriteUserPrompt, decideCandidates, dictionaryBlock, dictionaryVocabulary, findDictionaryCandidates, rewriteLines, rewriteTranscript, };
|
|
17
|
+
/** Correction is on when asked for, or by default when a dictionary or a rewrite is configured. */
|
|
18
|
+
export function isCorrectionEnabled(dictionary, options) {
|
|
19
|
+
if (options?.enabled !== undefined) {
|
|
20
|
+
return options.enabled;
|
|
21
|
+
}
|
|
22
|
+
// A `correction` object of any kind is a request for correction (its
|
|
23
|
+
// transliterate/punctuate settings would be dead otherwise); only
|
|
24
|
+
// `rewrite: false` without a dictionary leaves nothing to do.
|
|
25
|
+
return (Boolean(dictionary?.length) ||
|
|
26
|
+
(options !== undefined && options.rewrite !== false));
|
|
27
|
+
}
|
|
28
|
+
/** Run the guard (or skip it) and apply the dictionary to `text`. */
|
|
29
|
+
async function guardAndApply(text, dictionary, options, deps) {
|
|
30
|
+
const candidates = findDictionaryCandidates(text, dictionary);
|
|
31
|
+
if (candidates.length === 0) {
|
|
32
|
+
return { text, decisions: [] };
|
|
33
|
+
}
|
|
34
|
+
const started = Date.now();
|
|
35
|
+
const decisions = options?.guard === "none"
|
|
36
|
+
? candidates.map((candidate) => ({
|
|
37
|
+
heard: candidate.heard,
|
|
38
|
+
term: candidate.entry.term.trim(),
|
|
39
|
+
choice: "term",
|
|
40
|
+
note: "guard off",
|
|
41
|
+
}))
|
|
42
|
+
: await decideCandidates(text, candidates, deps.decide, {
|
|
43
|
+
context: options?.context,
|
|
44
|
+
timeoutMs: options?.decideTimeoutMs ?? DEFAULT_DECIDE_TIMEOUT_MS,
|
|
45
|
+
});
|
|
46
|
+
const ms = Date.now() - started;
|
|
47
|
+
const applied = decisions.filter((d) => d.choice === "term").length;
|
|
48
|
+
return {
|
|
49
|
+
text: applyDictionary(text, candidates, decisions),
|
|
50
|
+
decisions,
|
|
51
|
+
step: `dictionary ${applied}/${candidates.length} (${ms} ms)`,
|
|
52
|
+
decideMs: options?.guard === "none" ? undefined : ms,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Correct one transcript (a whole short recording, or one final utterance
|
|
57
|
+
* when streaming). Never throws on a layer's failure and never returns empty
|
|
58
|
+
* text when either transcript had words.
|
|
59
|
+
*
|
|
60
|
+
* When the primary engine reported a low language score
|
|
61
|
+
* (`languageDetected === false`) and a second opinion exists, the second
|
|
62
|
+
* opinion leads: an Indic engine on English speech produces confident-looking
|
|
63
|
+
* nonsense, and the English-oriented engine's text is the better base. The
|
|
64
|
+
* engine text then goes to the rewrite as the secondary transcript, so
|
|
65
|
+
* Indian-language words it did catch are not lost.
|
|
66
|
+
*/
|
|
67
|
+
export async function correctTranscript(input, deps) {
|
|
68
|
+
const started = Date.now();
|
|
69
|
+
const { dictionary, options } = input;
|
|
70
|
+
const steps = [];
|
|
71
|
+
const timings = {};
|
|
72
|
+
const engineText = input.text ?? "";
|
|
73
|
+
const second = input.secondOpinion?.trim() ? input.secondOpinion : undefined;
|
|
74
|
+
let lead = engineText;
|
|
75
|
+
let secondary = second;
|
|
76
|
+
let secondLed = false;
|
|
77
|
+
if (second && input.languageDetected === false) {
|
|
78
|
+
lead = second;
|
|
79
|
+
secondary = engineText.trim() ? engineText : undefined;
|
|
80
|
+
secondLed = true;
|
|
81
|
+
steps.push("second opinion led (primary unsure)");
|
|
82
|
+
}
|
|
83
|
+
else if (second && !engineText.trim()) {
|
|
84
|
+
lead = second;
|
|
85
|
+
secondary = undefined;
|
|
86
|
+
secondLed = true;
|
|
87
|
+
steps.push("second opinion led (primary empty)");
|
|
88
|
+
}
|
|
89
|
+
if (!isCorrectionEnabled(dictionary, options) || !lead.trim()) {
|
|
90
|
+
return {
|
|
91
|
+
text: lead,
|
|
92
|
+
decisions: [],
|
|
93
|
+
steps,
|
|
94
|
+
timings: { correctionMs: Date.now() - started },
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
const guarded = await guardAndApply(lead, dictionary, options, deps);
|
|
98
|
+
let text = guarded.text;
|
|
99
|
+
if (guarded.step) {
|
|
100
|
+
steps.push(guarded.step);
|
|
101
|
+
}
|
|
102
|
+
if (guarded.decideMs !== undefined) {
|
|
103
|
+
timings.decideMs = guarded.decideMs;
|
|
104
|
+
}
|
|
105
|
+
if (options?.rewrite !== false) {
|
|
106
|
+
const outcome = await rewriteTranscript({
|
|
107
|
+
primary: text,
|
|
108
|
+
secondOpinion: secondary,
|
|
109
|
+
dictionary,
|
|
110
|
+
options,
|
|
111
|
+
labels: secondLed
|
|
112
|
+
? {
|
|
113
|
+
primary: "second engine",
|
|
114
|
+
secondary: "primary engine, unsure of the language",
|
|
115
|
+
}
|
|
116
|
+
: undefined,
|
|
117
|
+
literal: guarded.decisions
|
|
118
|
+
.filter((d) => d.choice === "literal")
|
|
119
|
+
.map((d) => d.heard),
|
|
120
|
+
onPartial: input.onPartial,
|
|
121
|
+
}, deps.rewrite);
|
|
122
|
+
timings.rewriteMs = outcome.ms;
|
|
123
|
+
switch (outcome.status) {
|
|
124
|
+
case "ok":
|
|
125
|
+
text = outcome.text;
|
|
126
|
+
steps.push(`rewrite ${outcome.ms} ms`);
|
|
127
|
+
break;
|
|
128
|
+
case "timeout":
|
|
129
|
+
steps.push("rewrite timed out · kept text");
|
|
130
|
+
break;
|
|
131
|
+
case "error":
|
|
132
|
+
steps.push(`rewrite failed (${outcome.error ?? "error"}) · kept text`);
|
|
133
|
+
break;
|
|
134
|
+
case "empty":
|
|
135
|
+
steps.push("rewrite returned nothing · kept text");
|
|
136
|
+
break;
|
|
137
|
+
case "dropped":
|
|
138
|
+
steps.push(`rewrite dropped ${Math.round((outcome.dropRatio ?? 0) * 100)}% of words · kept text`);
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
timings.correctionMs = Date.now() - started;
|
|
143
|
+
return {
|
|
144
|
+
text: text.trim() ? text : lead,
|
|
145
|
+
decisions: guarded.decisions,
|
|
146
|
+
steps,
|
|
147
|
+
timings,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Correct a long transcript line by line (file transcription, diarized
|
|
152
|
+
* turns). With the rewrite on, lines go to the model in numbered batches of
|
|
153
|
+
* 25 and the dictionary travels in the prompt — the model sees each alias in
|
|
154
|
+
* its sentence, which is the context the guard would otherwise supply. With
|
|
155
|
+
* `rewrite: false`, each line with candidates gets its own guard call (up to
|
|
156
|
+
* four at a time) and the dictionary is applied per line. The result always
|
|
157
|
+
* has exactly as many lines as the input.
|
|
158
|
+
*/
|
|
159
|
+
export async function correctTranscriptLines(lines, dictionary, options, deps) {
|
|
160
|
+
const started = Date.now();
|
|
161
|
+
if (!isCorrectionEnabled(dictionary, options) || lines.length === 0) {
|
|
162
|
+
return { lines: [...lines], decisions: [], steps: [], timings: {} };
|
|
163
|
+
}
|
|
164
|
+
if (options?.rewrite !== false) {
|
|
165
|
+
const outcome = await rewriteLines(lines, deps.rewrite, {
|
|
166
|
+
dictionary,
|
|
167
|
+
correction: options,
|
|
168
|
+
});
|
|
169
|
+
const steps = [
|
|
170
|
+
`rewrite ${outcome.batchesOk}/${outcome.batches} batches (${outcome.ms} ms)`,
|
|
171
|
+
];
|
|
172
|
+
if (outcome.batchesOk < outcome.batches) {
|
|
173
|
+
steps.push(`${outcome.batches - outcome.batchesOk} batch(es) failed · kept their lines`);
|
|
174
|
+
}
|
|
175
|
+
if (outcome.linesKept > 0) {
|
|
176
|
+
steps.push(`${outcome.linesKept} line(s) dropped words · kept their text`);
|
|
177
|
+
}
|
|
178
|
+
return {
|
|
179
|
+
lines: outcome.lines,
|
|
180
|
+
decisions: [],
|
|
181
|
+
steps,
|
|
182
|
+
timings: { rewriteMs: outcome.ms, correctionMs: Date.now() - started },
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
const out = [...lines];
|
|
186
|
+
// Per line, so the flattened decisions follow line order whatever finishes first.
|
|
187
|
+
const perLine = lines.map(() => []);
|
|
188
|
+
let candidatesSeen = 0;
|
|
189
|
+
let applied = 0;
|
|
190
|
+
let next = 0;
|
|
191
|
+
const worker = async () => {
|
|
192
|
+
while (next < lines.length) {
|
|
193
|
+
const index = next++;
|
|
194
|
+
const guarded = await guardAndApply(lines[index], dictionary, options, deps);
|
|
195
|
+
out[index] = guarded.text;
|
|
196
|
+
perLine[index] = guarded.decisions;
|
|
197
|
+
candidatesSeen += guarded.decisions.length;
|
|
198
|
+
applied += guarded.decisions.filter((d) => d.choice === "term").length;
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
await Promise.all(Array.from({ length: Math.min(4, lines.length) }, () => worker()));
|
|
202
|
+
const ms = Date.now() - started;
|
|
203
|
+
return {
|
|
204
|
+
lines: out,
|
|
205
|
+
decisions: perLine.flat(),
|
|
206
|
+
steps: candidatesSeen
|
|
207
|
+
? [`dictionary ${applied}/${candidatesSeen} (${ms} ms)`]
|
|
208
|
+
: [],
|
|
209
|
+
timings: { decideMs: ms, correctionMs: ms },
|
|
210
|
+
};
|
|
211
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM rewrite of a transcript: spelling, punctuation, dictionary terms and
|
|
3
|
+
* transliteration, for English, any Indian language, or a mix of the two.
|
|
4
|
+
*
|
|
5
|
+
* The rewrite is a convenience, never a risk to the words:
|
|
6
|
+
* - on timeout (default 12 s) or error the input text is kept;
|
|
7
|
+
* - a rewrite that drops more than `maxDropRatio` (default 0.25) of the words
|
|
8
|
+
* is discarded — a model that summarises or truncates must not cost words;
|
|
9
|
+
* - the model's answer is the last non-empty line(s), because some models
|
|
10
|
+
* echo a preamble ("Here is the corrected sentence:") before it;
|
|
11
|
+
* - `maxTokens` is always an integer: a float (`60 + 1.5 × chars` unrounded)
|
|
12
|
+
* was once rejected with an HTTP 400.
|
|
13
|
+
*
|
|
14
|
+
* The prompt's hardest rule is "never translate": asked to "correct" a
|
|
15
|
+
* Hindi-English sentence, models readily return an English translation, which
|
|
16
|
+
* reads well and loses what was said.
|
|
17
|
+
*
|
|
18
|
+
* @module voice/correction/rewrite
|
|
19
|
+
*/
|
|
20
|
+
import type { STTCorrectionDeps, STTCorrectionOptions, STTDictionaryEntry, STTRewriteLinesOutcome, STTRewriteOutcome } from "../../types/index.js";
|
|
21
|
+
/** Default per-call rewrite timeout; on expiry the input text is kept. */
|
|
22
|
+
export declare const DEFAULT_REWRITE_TIMEOUT_MS = 12000;
|
|
23
|
+
/** Default per-batch timeout for `rewriteLines`: a batch carries up to 25 lines. */
|
|
24
|
+
export declare const DEFAULT_REWRITE_LINES_TIMEOUT_MS = 60000;
|
|
25
|
+
/** Default share of words a rewrite may drop before it is discarded. */
|
|
26
|
+
export declare const DEFAULT_MAX_DROP_RATIO = 0.25;
|
|
27
|
+
/** Lines per request in `rewriteLines`. */
|
|
28
|
+
export declare const REWRITE_LINES_BATCH = 25;
|
|
29
|
+
/**
|
|
30
|
+
* Output budget: ~1.5 tokens per input character plus a fixed allowance,
|
|
31
|
+
* capped. Generous on purpose — transliterated Indic text takes more tokens
|
|
32
|
+
* than its source script — and rounded, because a float is a 400 on at least
|
|
33
|
+
* one OpenAI-compatible server.
|
|
34
|
+
*/
|
|
35
|
+
export declare function rewriteMaxTokens(inputChars: number): number;
|
|
36
|
+
/** Words for the drop guard: whitespace tokens that hold a letter or a digit. */
|
|
37
|
+
export declare function countWords(text: string): number;
|
|
38
|
+
/**
|
|
39
|
+
* System prompt for the rewrite. `lines: true` is the batched variant for long
|
|
40
|
+
* transcripts: numbered lines in, the same numbered lines out.
|
|
41
|
+
*/
|
|
42
|
+
export declare function buildRewriteSystemPrompt(options?: {
|
|
43
|
+
transliterate?: "latin" | "native";
|
|
44
|
+
punctuate?: boolean;
|
|
45
|
+
dictionary?: readonly STTDictionaryEntry[];
|
|
46
|
+
lines?: boolean;
|
|
47
|
+
}): string;
|
|
48
|
+
/**
|
|
49
|
+
* User message for the rewrite. With a second opinion the two transcripts are
|
|
50
|
+
* labelled; `literal` lists aliases the guard decided were meant literally, so
|
|
51
|
+
* the rewrite does not turn them back into dictionary terms.
|
|
52
|
+
*/
|
|
53
|
+
export declare function buildRewriteUserPrompt(primary: string, secondOpinion?: string, extras?: {
|
|
54
|
+
labels?: {
|
|
55
|
+
primary?: string;
|
|
56
|
+
secondary?: string;
|
|
57
|
+
};
|
|
58
|
+
literal?: readonly string[];
|
|
59
|
+
}): string;
|
|
60
|
+
/**
|
|
61
|
+
* The model's answer: the last `lineCount` non-empty lines, since a preamble,
|
|
62
|
+
* if any, comes first. An echoed "Transcript:" label is stripped.
|
|
63
|
+
*/
|
|
64
|
+
export declare function extractRewrite(content: string, lineCount?: number): string;
|
|
65
|
+
/**
|
|
66
|
+
* Rewrite one transcript (a sentence, an utterance, or a short multi-line
|
|
67
|
+
* text). `primary` is the leading text — already dictionary-applied — and the
|
|
68
|
+
* one the drop guard measures against. Never throws.
|
|
69
|
+
*/
|
|
70
|
+
export declare function rewriteTranscript(input: {
|
|
71
|
+
primary: string;
|
|
72
|
+
secondOpinion?: string;
|
|
73
|
+
dictionary?: readonly STTDictionaryEntry[];
|
|
74
|
+
options?: STTCorrectionOptions;
|
|
75
|
+
labels?: {
|
|
76
|
+
primary?: string;
|
|
77
|
+
secondary?: string;
|
|
78
|
+
};
|
|
79
|
+
literal?: readonly string[];
|
|
80
|
+
onPartial?: (text: string) => void;
|
|
81
|
+
}, rewrite: STTCorrectionDeps["rewrite"]): Promise<STTRewriteOutcome>;
|
|
82
|
+
/**
|
|
83
|
+
* Batched rewrite for long transcripts (file transcription, diarized turns):
|
|
84
|
+
* 25 numbered lines per request, the same numbers back. A failed or timed-out
|
|
85
|
+
* batch keeps its lines; a line missing from the output, empty, or shorter by
|
|
86
|
+
* more than `maxDropRatio` of its words keeps its input. Blank input lines
|
|
87
|
+
* are never sent and come back as they were. Never throws.
|
|
88
|
+
*/
|
|
89
|
+
export declare function rewriteLines(lines: readonly string[], rewrite: STTCorrectionDeps["rewrite"], options?: {
|
|
90
|
+
dictionary?: readonly STTDictionaryEntry[];
|
|
91
|
+
correction?: STTCorrectionOptions;
|
|
92
|
+
}): Promise<STTRewriteLinesOutcome>;
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM rewrite of a transcript: spelling, punctuation, dictionary terms and
|
|
3
|
+
* transliteration, for English, any Indian language, or a mix of the two.
|
|
4
|
+
*
|
|
5
|
+
* The rewrite is a convenience, never a risk to the words:
|
|
6
|
+
* - on timeout (default 12 s) or error the input text is kept;
|
|
7
|
+
* - a rewrite that drops more than `maxDropRatio` (default 0.25) of the words
|
|
8
|
+
* is discarded — a model that summarises or truncates must not cost words;
|
|
9
|
+
* - the model's answer is the last non-empty line(s), because some models
|
|
10
|
+
* echo a preamble ("Here is the corrected sentence:") before it;
|
|
11
|
+
* - `maxTokens` is always an integer: a float (`60 + 1.5 × chars` unrounded)
|
|
12
|
+
* was once rejected with an HTTP 400.
|
|
13
|
+
*
|
|
14
|
+
* The prompt's hardest rule is "never translate": asked to "correct" a
|
|
15
|
+
* Hindi-English sentence, models readily return an English translation, which
|
|
16
|
+
* reads well and loses what was said.
|
|
17
|
+
*
|
|
18
|
+
* @module voice/correction/rewrite
|
|
19
|
+
*/
|
|
20
|
+
import { TimeoutError, withTimeout } from "../../utils/async/withTimeout.js";
|
|
21
|
+
import { dictionaryBlock } from "./dictionary.js";
|
|
22
|
+
/** Default per-call rewrite timeout; on expiry the input text is kept. */
|
|
23
|
+
export const DEFAULT_REWRITE_TIMEOUT_MS = 12000;
|
|
24
|
+
/** Default per-batch timeout for `rewriteLines`: a batch carries up to 25 lines. */
|
|
25
|
+
export const DEFAULT_REWRITE_LINES_TIMEOUT_MS = 60000;
|
|
26
|
+
/** Default share of words a rewrite may drop before it is discarded. */
|
|
27
|
+
export const DEFAULT_MAX_DROP_RATIO = 0.25;
|
|
28
|
+
/** Lines per request in `rewriteLines`. */
|
|
29
|
+
export const REWRITE_LINES_BATCH = 25;
|
|
30
|
+
/** Batches of `rewriteLines` in flight at once. */
|
|
31
|
+
const REWRITE_LINES_CONCURRENCY = 4;
|
|
32
|
+
/** Ceiling on `maxTokens` for any one rewrite request. */
|
|
33
|
+
const MAX_REWRITE_TOKENS = 4096;
|
|
34
|
+
/**
|
|
35
|
+
* Output budget: ~1.5 tokens per input character plus a fixed allowance,
|
|
36
|
+
* capped. Generous on purpose — transliterated Indic text takes more tokens
|
|
37
|
+
* than its source script — and rounded, because a float is a 400 on at least
|
|
38
|
+
* one OpenAI-compatible server.
|
|
39
|
+
*/
|
|
40
|
+
export function rewriteMaxTokens(inputChars) {
|
|
41
|
+
return Math.min(MAX_REWRITE_TOKENS, Math.max(1, Math.ceil(60 + 1.5 * Math.max(0, inputChars))));
|
|
42
|
+
}
|
|
43
|
+
/** Words for the drop guard: whitespace tokens that hold a letter or a digit. */
|
|
44
|
+
export function countWords(text) {
|
|
45
|
+
return text.split(/\s+/u).filter((token) => /[\p{L}\p{N}]/u.test(token))
|
|
46
|
+
.length;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The script rule. Native-script examples appear only for `"latin"`, where
|
|
50
|
+
* they show the conversion; with `"native"` they would invite the model to
|
|
51
|
+
* convert anyway.
|
|
52
|
+
*/
|
|
53
|
+
function scriptRule(transliterate) {
|
|
54
|
+
if (transliterate === "native") {
|
|
55
|
+
return ("- Indian-language words stay in that language AND in their own script " +
|
|
56
|
+
"(Devanagari, Bengali, Gurmukhi, Gujarati, Odia, Tamil, Telugu, Kannada, " +
|
|
57
|
+
"Malayalam, Urdu …); never rewrite them in Latin letters. Fix only their spelling.");
|
|
58
|
+
}
|
|
59
|
+
return ("- Indian-language words stay in that language but are ALWAYS written in Latin " +
|
|
60
|
+
"letters the way people type on phones (Hinglish, Tanglish, Kanglish …): " +
|
|
61
|
+
"मेरा नाम → mera naam; भेज देना → bhej dena; " +
|
|
62
|
+
"நாளைக்கு மீட்டிங் இருக்கு → naalaikku meeting irukku; ಇವತ್ತು → ivattu; " +
|
|
63
|
+
"నేను వస్తాను → nenu vastanu; আমি আসছি → ami aschi. " +
|
|
64
|
+
"No Indic script may remain in the output.");
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* System prompt for the rewrite. `lines: true` is the batched variant for long
|
|
68
|
+
* transcripts: numbered lines in, the same numbered lines out.
|
|
69
|
+
*/
|
|
70
|
+
export function buildRewriteSystemPrompt(options) {
|
|
71
|
+
const transliterate = options?.transliterate ?? "latin";
|
|
72
|
+
const punctuate = options?.punctuate ?? true;
|
|
73
|
+
const lines = options?.lines ?? false;
|
|
74
|
+
const table = dictionaryBlock(options?.dictionary);
|
|
75
|
+
const rules = [
|
|
76
|
+
scriptRule(transliterate),
|
|
77
|
+
"- NEVER translate. An Indian-language word stays in its language and an English word stays English.",
|
|
78
|
+
"- English words get their correct English spelling (please, before, client, deployment, meeting). " +
|
|
79
|
+
"Never leave an English word spelled phonetically, in any script.",
|
|
80
|
+
];
|
|
81
|
+
if (table) {
|
|
82
|
+
rules.push('- Use the dictionary: a word or phrase that sounds like a term or one of its "also heard as" forms, ' +
|
|
83
|
+
"and means that term here, is written as the term exactly.");
|
|
84
|
+
}
|
|
85
|
+
if (lines) {
|
|
86
|
+
rules.push("- Also fix near-homophones that make no sense in context when the intended word is obvious.");
|
|
87
|
+
}
|
|
88
|
+
rules.push(punctuate
|
|
89
|
+
? "- Add natural punctuation and capitalization; write numbers as digits; keep fillers."
|
|
90
|
+
: "- Do not add or change punctuation or capitalization; keep fillers.");
|
|
91
|
+
rules.push(lines
|
|
92
|
+
? "- Keep every word: never summarize, add, drop, reorder or merge. Keep any speaker label at the " +
|
|
93
|
+
"start of a line unchanged. A line that needs no change is returned verbatim."
|
|
94
|
+
: "- Keep every word: never summarize, add, drop or reorder. With two transcripts, take " +
|
|
95
|
+
"Indian-language words from the one that has them in an Indian language and English words " +
|
|
96
|
+
"from the one that spells them in English; elsewhere prefer Transcript 1.");
|
|
97
|
+
const head = [
|
|
98
|
+
"You are a post-editor for speech-recognition (ASR) transcripts. The speaker talks English, " +
|
|
99
|
+
"an Indian language (Hindi, Bengali, Marathi, Gujarati, Punjabi, Odia, Assamese, Urdu, Tamil, " +
|
|
100
|
+
"Telugu, Kannada, Malayalam, …), or mixes an Indian language with English.",
|
|
101
|
+
lines
|
|
102
|
+
? "Input: a numbered list of transcript lines, possibly in an Indic script (such engines write " +
|
|
103
|
+
"English words phonetically in that script)."
|
|
104
|
+
: "Input: an ASR transcript, possibly in an Indic script (such engines write English words " +
|
|
105
|
+
"phonetically in that script), and possibly a second transcript of the same audio from another engine.",
|
|
106
|
+
"Write the speaker's words exactly as they said them:",
|
|
107
|
+
...rules,
|
|
108
|
+
lines
|
|
109
|
+
? "Output: the same numbered lines, one per input line, nothing else."
|
|
110
|
+
: "Output only the corrected text, nothing else.",
|
|
111
|
+
].join("\n");
|
|
112
|
+
return table ? `${head}\n\n${table}` : head;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* User message for the rewrite. With a second opinion the two transcripts are
|
|
116
|
+
* labelled; `literal` lists aliases the guard decided were meant literally, so
|
|
117
|
+
* the rewrite does not turn them back into dictionary terms.
|
|
118
|
+
*/
|
|
119
|
+
export function buildRewriteUserPrompt(primary, secondOpinion, extras) {
|
|
120
|
+
const second = secondOpinion?.trim();
|
|
121
|
+
const body = second
|
|
122
|
+
? `Transcript 1 (${extras?.labels?.primary ?? "primary engine"}): ${primary}\n` +
|
|
123
|
+
`Transcript 2 (${extras?.labels?.secondary ?? "second engine"}): ${second}`
|
|
124
|
+
: `Transcript: ${primary}`;
|
|
125
|
+
const literal = [...new Set(extras?.literal ?? [])].filter(Boolean);
|
|
126
|
+
return literal.length
|
|
127
|
+
? `${body}\nMeant literally here, not dictionary terms: ${literal.map((w) => `"${w}"`).join(", ")}`
|
|
128
|
+
: body;
|
|
129
|
+
}
|
|
130
|
+
/** A label the model may echo in front of its answer ("Transcript 1 (…):"). */
|
|
131
|
+
const ECHOED_LABEL_RE = /^(?:corrected\s+)?transcript(?:\s*\d+)?(?:\s*\([^)]*\))?\s*:\s*/iu;
|
|
132
|
+
/**
|
|
133
|
+
* The model's answer: the last `lineCount` non-empty lines, since a preamble,
|
|
134
|
+
* if any, comes first. An echoed "Transcript:" label is stripped.
|
|
135
|
+
*/
|
|
136
|
+
export function extractRewrite(content, lineCount = 1) {
|
|
137
|
+
const lines = content
|
|
138
|
+
.split(/\r?\n/u)
|
|
139
|
+
.map((line) => line.trim())
|
|
140
|
+
.filter(Boolean);
|
|
141
|
+
return lines
|
|
142
|
+
.slice(-Math.max(1, lineCount))
|
|
143
|
+
.map((line) => line.replace(ECHOED_LABEL_RE, "").trim())
|
|
144
|
+
.join("\n")
|
|
145
|
+
.trim();
|
|
146
|
+
}
|
|
147
|
+
/** Share of `before`'s words missing from `after`, 0 when nothing was dropped. */
|
|
148
|
+
function dropRatio(before, after) {
|
|
149
|
+
const inWords = countWords(before);
|
|
150
|
+
if (inWords === 0) {
|
|
151
|
+
return 0;
|
|
152
|
+
}
|
|
153
|
+
return Math.max(0, (inWords - countWords(after)) / inWords);
|
|
154
|
+
}
|
|
155
|
+
function resolveDropRatio(value) {
|
|
156
|
+
return typeof value === "number" && Number.isFinite(value)
|
|
157
|
+
? Math.min(1, Math.max(0, value))
|
|
158
|
+
: DEFAULT_MAX_DROP_RATIO;
|
|
159
|
+
}
|
|
160
|
+
function errorMessage(error) {
|
|
161
|
+
return error instanceof Error ? error.message : String(error);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Rewrite one transcript (a sentence, an utterance, or a short multi-line
|
|
165
|
+
* text). `primary` is the leading text — already dictionary-applied — and the
|
|
166
|
+
* one the drop guard measures against. Never throws.
|
|
167
|
+
*/
|
|
168
|
+
export async function rewriteTranscript(input, rewrite) {
|
|
169
|
+
const started = Date.now();
|
|
170
|
+
const { primary, secondOpinion, options } = input;
|
|
171
|
+
const settings = options?.rewrite === false ? undefined : options?.rewrite;
|
|
172
|
+
const timeoutMs = settings?.timeoutMs && settings.timeoutMs > 0
|
|
173
|
+
? settings.timeoutMs
|
|
174
|
+
: DEFAULT_REWRITE_TIMEOUT_MS;
|
|
175
|
+
const lineCount = Math.max(1, primary.split(/\r?\n/u).filter((line) => line.trim()).length);
|
|
176
|
+
// Partials after a timeout would overwrite text the caller already settled.
|
|
177
|
+
let live = true;
|
|
178
|
+
const onPartial = input.onPartial
|
|
179
|
+
? (partial) => {
|
|
180
|
+
if (live) {
|
|
181
|
+
const text = extractRewrite(partial, lineCount);
|
|
182
|
+
if (text) {
|
|
183
|
+
input.onPartial?.(text);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
: undefined;
|
|
188
|
+
const keep = (status, extra) => ({
|
|
189
|
+
text: primary,
|
|
190
|
+
status,
|
|
191
|
+
ms: Date.now() - started,
|
|
192
|
+
...extra,
|
|
193
|
+
});
|
|
194
|
+
let content;
|
|
195
|
+
try {
|
|
196
|
+
content = await withTimeout(rewrite({
|
|
197
|
+
system: buildRewriteSystemPrompt({
|
|
198
|
+
transliterate: options?.transliterate,
|
|
199
|
+
punctuate: options?.punctuate,
|
|
200
|
+
dictionary: input.dictionary,
|
|
201
|
+
}),
|
|
202
|
+
user: buildRewriteUserPrompt(primary, secondOpinion, {
|
|
203
|
+
labels: input.labels,
|
|
204
|
+
literal: input.literal,
|
|
205
|
+
}),
|
|
206
|
+
provider: settings?.provider,
|
|
207
|
+
model: settings?.model,
|
|
208
|
+
maxTokens: rewriteMaxTokens(primary.length + (secondOpinion?.length ?? 0)),
|
|
209
|
+
timeoutMs,
|
|
210
|
+
onPartial,
|
|
211
|
+
}), timeoutMs);
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
live = false;
|
|
215
|
+
return error instanceof TimeoutError
|
|
216
|
+
? keep("timeout")
|
|
217
|
+
: keep("error", { error: errorMessage(error) });
|
|
218
|
+
}
|
|
219
|
+
live = false;
|
|
220
|
+
const text = extractRewrite(typeof content === "string" ? content : "", lineCount);
|
|
221
|
+
if (!text) {
|
|
222
|
+
return keep("empty");
|
|
223
|
+
}
|
|
224
|
+
const ratio = dropRatio(primary, text);
|
|
225
|
+
if (ratio > resolveDropRatio(options?.maxDropRatio)) {
|
|
226
|
+
return keep("dropped", { dropRatio: ratio });
|
|
227
|
+
}
|
|
228
|
+
return { text, status: "ok", ms: Date.now() - started };
|
|
229
|
+
}
|
|
230
|
+
/** Run `task` over `items` with at most `limit` in flight; results keep input order. */
|
|
231
|
+
async function mapLimited(items, limit, task) {
|
|
232
|
+
const results = new Array(items.length);
|
|
233
|
+
let next = 0;
|
|
234
|
+
const worker = async () => {
|
|
235
|
+
while (next < items.length) {
|
|
236
|
+
const index = next++;
|
|
237
|
+
results[index] = await task(items[index], index);
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
241
|
+
return results;
|
|
242
|
+
}
|
|
243
|
+
/** `1. text` / `1) text` in the model's numbered output. */
|
|
244
|
+
const NUMBERED_LINE_RE = /^\s*(\d+)[.)]\s?(.*)$/gmu;
|
|
245
|
+
/**
|
|
246
|
+
* Batched rewrite for long transcripts (file transcription, diarized turns):
|
|
247
|
+
* 25 numbered lines per request, the same numbers back. A failed or timed-out
|
|
248
|
+
* batch keeps its lines; a line missing from the output, empty, or shorter by
|
|
249
|
+
* more than `maxDropRatio` of its words keeps its input. Blank input lines
|
|
250
|
+
* are never sent and come back as they were. Never throws.
|
|
251
|
+
*/
|
|
252
|
+
export async function rewriteLines(lines, rewrite, options) {
|
|
253
|
+
const started = Date.now();
|
|
254
|
+
const out = [...lines];
|
|
255
|
+
const settings = options?.correction?.rewrite === false
|
|
256
|
+
? undefined
|
|
257
|
+
: options?.correction?.rewrite;
|
|
258
|
+
const timeoutMs = settings?.timeoutMs && settings.timeoutMs > 0
|
|
259
|
+
? settings.timeoutMs
|
|
260
|
+
: DEFAULT_REWRITE_LINES_TIMEOUT_MS;
|
|
261
|
+
const maxDrop = resolveDropRatio(options?.correction?.maxDropRatio);
|
|
262
|
+
const system = buildRewriteSystemPrompt({
|
|
263
|
+
transliterate: options?.correction?.transliterate,
|
|
264
|
+
punctuate: options?.correction?.punctuate,
|
|
265
|
+
dictionary: options?.dictionary,
|
|
266
|
+
lines: true,
|
|
267
|
+
});
|
|
268
|
+
const indices = lines
|
|
269
|
+
.map((line, index) => (line.trim() ? index : -1))
|
|
270
|
+
.filter((index) => index >= 0);
|
|
271
|
+
const batches = [];
|
|
272
|
+
for (let i = 0; i < indices.length; i += REWRITE_LINES_BATCH) {
|
|
273
|
+
batches.push(indices.slice(i, i + REWRITE_LINES_BATCH));
|
|
274
|
+
}
|
|
275
|
+
let linesKept = 0;
|
|
276
|
+
const okFlags = await mapLimited(batches, REWRITE_LINES_CONCURRENCY, async (batch) => {
|
|
277
|
+
const numbered = batch
|
|
278
|
+
.map((index, k) => `${k + 1}. ${lines[index].trim()}`)
|
|
279
|
+
.join("\n");
|
|
280
|
+
let content;
|
|
281
|
+
try {
|
|
282
|
+
content = await withTimeout(rewrite({
|
|
283
|
+
system,
|
|
284
|
+
user: numbered,
|
|
285
|
+
provider: settings?.provider,
|
|
286
|
+
model: settings?.model,
|
|
287
|
+
maxTokens: rewriteMaxTokens(numbered.length),
|
|
288
|
+
timeoutMs,
|
|
289
|
+
}), timeoutMs);
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
return false;
|
|
293
|
+
}
|
|
294
|
+
const answers = new Map();
|
|
295
|
+
for (const match of String(content ?? "").matchAll(NUMBERED_LINE_RE)) {
|
|
296
|
+
const k = Number(match[1]) - 1;
|
|
297
|
+
const text = match[2].trim();
|
|
298
|
+
if (k >= 0 && k < batch.length && text && !answers.has(k)) {
|
|
299
|
+
answers.set(k, text);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
batch.forEach((index, k) => {
|
|
303
|
+
const text = answers.get(k);
|
|
304
|
+
if (!text) {
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
if (dropRatio(lines[index], text) > maxDrop) {
|
|
308
|
+
linesKept += 1;
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
out[index] = text;
|
|
312
|
+
});
|
|
313
|
+
return answers.size > 0;
|
|
314
|
+
});
|
|
315
|
+
return {
|
|
316
|
+
lines: out,
|
|
317
|
+
batchesOk: okFlags.filter(Boolean).length,
|
|
318
|
+
batches: batches.length,
|
|
319
|
+
linesKept,
|
|
320
|
+
ms: Date.now() - started,
|
|
321
|
+
};
|
|
322
|
+
}
|
package/dist/voice/errors.d.ts
CHANGED
|
@@ -51,6 +51,21 @@ export declare class STTError extends VoiceError {
|
|
|
51
51
|
* Create an error for unsupported provider
|
|
52
52
|
*/
|
|
53
53
|
static providerNotSupported(provider: string, availableProviders?: string[]): STTError;
|
|
54
|
+
/**
|
|
55
|
+
* Create an error for a provider name that is neither a shipped STT
|
|
56
|
+
* provider nor a registered handler. Lists what is configured and what
|
|
57
|
+
* would configure the rest, so the caller can act on the message alone.
|
|
58
|
+
*/
|
|
59
|
+
static unknownProvider(provider: string, configuredProviders: string[], howToConfigure?: string): STTError;
|
|
60
|
+
/**
|
|
61
|
+
* Create an error for audio that could not be read (a missing file, a URL
|
|
62
|
+
* that failed, an input of the wrong kind).
|
|
63
|
+
*/
|
|
64
|
+
static audioLoadFailed(reason: string, originalError?: Error): STTError;
|
|
65
|
+
/**
|
|
66
|
+
* Create an error for a stream of frames handed to a batch entry point.
|
|
67
|
+
*/
|
|
68
|
+
static streamInputNotSupported(): STTError;
|
|
54
69
|
/**
|
|
55
70
|
* Create an error for stream processing failure
|
|
56
71
|
*/
|