@juspay/neurolink 12.48.2 → 12.49.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/CHANGELOG.md +2 -2
  2. package/dist/browser/neurolink.min.js +616 -595
  3. package/dist/cli/commands/serve.d.ts +0 -3
  4. package/dist/cli/commands/serve.js +85 -3
  5. package/dist/cli/commands/transcribe.d.ts +24 -0
  6. package/dist/cli/commands/transcribe.js +718 -0
  7. package/dist/cli/factories/commandFactory.d.ts +10 -0
  8. package/dist/cli/factories/commandFactory.js +13 -0
  9. package/dist/cli/parser.js +3 -0
  10. package/dist/factories/mediaHandlerCatalog.js +11 -9
  11. package/dist/factories/providerDescriptors.js +5 -4
  12. package/dist/factories/sttDescriptors.d.ts +63 -0
  13. package/dist/factories/sttDescriptors.js +286 -0
  14. package/dist/index.d.ts +4 -1
  15. package/dist/index.js +7 -1
  16. package/dist/neurolink.d.ts +68 -1
  17. package/dist/neurolink.js +202 -4
  18. package/dist/processors/media/AudioProcessor.d.ts +15 -56
  19. package/dist/processors/media/AudioProcessor.js +105 -276
  20. package/dist/processors/media/VideoProcessor.d.ts +3 -5
  21. package/dist/processors/media/VideoProcessor.js +52 -51
  22. package/dist/providers/cloudflareClef.js +9 -1
  23. package/dist/server/abstract/baseServerAdapter.d.ts +7 -0
  24. package/dist/server/abstract/baseServerAdapter.js +13 -1
  25. package/dist/server/adapters/expressAdapter.d.ts +1 -0
  26. package/dist/server/adapters/expressAdapter.js +48 -4
  27. package/dist/server/adapters/fastifyAdapter.d.ts +1 -0
  28. package/dist/server/adapters/fastifyAdapter.js +38 -1
  29. package/dist/server/adapters/honoAdapter.d.ts +1 -4
  30. package/dist/server/adapters/honoAdapter.js +83 -1
  31. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  32. package/dist/server/adapters/koaAdapter.js +43 -0
  33. package/dist/server/index.d.ts +3 -2
  34. package/dist/server/index.js +3 -2
  35. package/dist/server/routes/index.d.ts +1 -0
  36. package/dist/server/routes/index.js +3 -0
  37. package/dist/server/routes/transcribeRoutes.d.ts +23 -0
  38. package/dist/server/routes/transcribeRoutes.js +506 -0
  39. package/dist/server/utils/rawBody.d.ts +32 -0
  40. package/dist/server/utils/rawBody.js +76 -0
  41. package/dist/server/utils/validation.d.ts +192 -0
  42. package/dist/server/utils/validation.js +100 -0
  43. package/dist/server/websocket/transcribeWebSocket.d.ts +42 -0
  44. package/dist/server/websocket/transcribeWebSocket.js +264 -0
  45. package/dist/types/cli.d.ts +46 -0
  46. package/dist/types/index.d.ts +1 -0
  47. package/dist/types/index.js +1 -0
  48. package/dist/types/providers.d.ts +7 -1
  49. package/dist/types/server.d.ts +117 -1
  50. package/dist/types/stt.d.ts +160 -7
  51. package/dist/types/transcribe.d.ts +506 -0
  52. package/dist/types/transcribe.js +29 -0
  53. package/dist/utils/speakerTurns.d.ts +28 -0
  54. package/dist/utils/speakerTurns.js +93 -0
  55. package/dist/utils/sttProcessor.d.ts +20 -2
  56. package/dist/utils/sttProcessor.js +73 -5
  57. package/dist/voice/correction/dictionary.d.ts +49 -0
  58. package/dist/voice/correction/dictionary.js +134 -0
  59. package/dist/voice/correction/guard.d.ts +35 -0
  60. package/dist/voice/correction/guard.js +161 -0
  61. package/dist/voice/correction/index.d.ts +47 -0
  62. package/dist/voice/correction/index.js +211 -0
  63. package/dist/voice/correction/rewrite.d.ts +92 -0
  64. package/dist/voice/correction/rewrite.js +322 -0
  65. package/dist/voice/errors.d.ts +15 -0
  66. package/dist/voice/errors.js +45 -0
  67. package/dist/voice/index.d.ts +20 -2
  68. package/dist/voice/index.js +75 -9
  69. package/dist/voice/providers/AzureSTT.d.ts +10 -7
  70. package/dist/voice/providers/AzureSTT.js +23 -75
  71. package/dist/voice/providers/DeepgramSTT.d.ts +17 -2
  72. package/dist/voice/providers/DeepgramSTT.js +109 -13
  73. package/dist/voice/providers/ElevenLabsSTT.d.ts +12 -2
  74. package/dist/voice/providers/ElevenLabsSTT.js +34 -8
  75. package/dist/voice/providers/GoogleSTT.d.ts +10 -8
  76. package/dist/voice/providers/GoogleSTT.js +21 -75
  77. package/dist/voice/providers/OpenAISTT.d.ts +30 -9
  78. package/dist/voice/providers/OpenAISTT.js +196 -21
  79. package/dist/voice/providers/WhistleSTT.d.ts +87 -0
  80. package/dist/voice/providers/WhistleSTT.js +313 -0
  81. package/dist/voice/streaming/chunkedStreamAdapter.d.ts +19 -0
  82. package/dist/voice/streaming/chunkedStreamAdapter.js +519 -0
  83. package/dist/voice/streaming/endpointer.d.ts +81 -0
  84. package/dist/voice/streaming/endpointer.js +247 -0
  85. package/dist/voice/streaming/localAgreement.d.ts +30 -0
  86. package/dist/voice/streaming/localAgreement.js +44 -0
  87. package/dist/voice/sttEnvCredentials.d.ts +12 -0
  88. package/dist/voice/sttEnvCredentials.js +12 -0
  89. package/dist/voice/transcribe.d.ts +47 -0
  90. package/dist/voice/transcribe.js +519 -0
  91. package/dist/voice/transcribeStream.d.ts +4 -0
  92. package/dist/voice/transcribeStream.js +584 -0
  93. package/dist/voice/whistle/assets.d.ts +47 -0
  94. package/dist/voice/whistle/assets.js +270 -0
  95. package/dist/voice/whistle/audio.d.ts +32 -0
  96. package/dist/voice/whistle/audio.js +267 -0
  97. package/dist/voice/whistle/engine.d.ts +32 -0
  98. package/dist/voice/whistle/engine.js +193 -0
  99. package/dist/voice/whistle/worker.d.ts +18 -0
  100. package/dist/voice/whistle/worker.js +294 -0
  101. package/docs-site/static/search-index.json +71 -12
  102. package/models/whistle/README.md +18 -0
  103. package/models/whistle/needle.js +2 -0
  104. package/models/whistle/needle.wasm +0 -0
  105. package/models/whistle/whistle.cact +0 -0
  106. package/package.json +7 -3
@@ -0,0 +1,76 @@
1
+ /**
2
+ * Raw request bodies and raw responses for the server adapters.
3
+ *
4
+ * The adapters parse JSON (and url-encoded forms) into `ctx.body`. Routes that
5
+ * take uploads — `POST /v1/audio/transcriptions` takes `multipart/form-data`
6
+ * the way OpenAI's API does — need the bytes instead, so for the content types
7
+ * below every adapter hands the route the unparsed body as a `Buffer`, bounded
8
+ * by `bodyParser.maxSize`. A route that must answer in a wire format other than
9
+ * the adapters' `{ data, metadata }` envelope (an OpenAI-compatible route, a
10
+ * plain-text response) returns a web `Response`, which every adapter sends as is.
11
+ *
12
+ * @module server/utils/rawBody
13
+ */
14
+ /** Content types whose body reaches the route as a `Buffer`. */
15
+ const RAW_BODY_CONTENT_TYPE = /^\s*(multipart\/form-data|application\/octet-stream|audio\/[\w.+-]+)\s*(;|$)/i;
16
+ /** `true` for a content type whose body the adapters pass through unparsed. */
17
+ export function isRawBodyContentType(contentType) {
18
+ return (typeof contentType === "string" && RAW_BODY_CONTENT_TYPE.test(contentType));
19
+ }
20
+ /** "10mb" / "512kb" / "1gb" / a byte count → bytes. Falls back to 10 MB. */
21
+ export function parseByteLimit(limit) {
22
+ if (typeof limit === "number" && Number.isFinite(limit) && limit > 0) {
23
+ return limit;
24
+ }
25
+ const match = typeof limit === "string"
26
+ ? limit.trim().match(/^(\d+(?:\.\d+)?)\s*(b|kb|mb|gb)?$/i)
27
+ : null;
28
+ if (!match) {
29
+ return 10 * 1024 * 1024;
30
+ }
31
+ const units = {
32
+ b: 1,
33
+ kb: 1024,
34
+ mb: 1024 * 1024,
35
+ gb: 1024 * 1024 * 1024,
36
+ };
37
+ return Math.floor(Number(match[1]) * units[(match[2] ?? "b").toLowerCase()]);
38
+ }
39
+ /** Thrown when a raw body grows past the adapter's `maxSize`. */
40
+ export class RawBodyTooLargeError extends Error {
41
+ status = 413;
42
+ constructor(limit) {
43
+ super(`Request body exceeds the server's ${limit}-byte limit`);
44
+ this.name = "RawBodyTooLargeError";
45
+ }
46
+ }
47
+ /** Read a Node request stream into one `Buffer`, refusing more than `maxBytes`. */
48
+ export async function readRawBody(stream, maxBytes) {
49
+ const chunks = [];
50
+ let total = 0;
51
+ for await (const chunk of stream) {
52
+ const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
53
+ total += buffer.length;
54
+ if (total > maxBytes) {
55
+ throw new RawBodyTooLargeError(maxBytes);
56
+ }
57
+ chunks.push(buffer);
58
+ }
59
+ return Buffer.concat(chunks);
60
+ }
61
+ /** A web `Response` returned by a route handler. */
62
+ export function isWebResponse(value) {
63
+ return typeof Response !== "undefined" && value instanceof Response;
64
+ }
65
+ /** Status, headers and body bytes of a web `Response`, for adapters that write to a Node response. */
66
+ export async function readWebResponse(response) {
67
+ const headers = {};
68
+ response.headers.forEach((value, key) => {
69
+ headers[key] = value;
70
+ });
71
+ return {
72
+ status: response.status,
73
+ headers,
74
+ body: Buffer.from(await response.arrayBuffer()),
75
+ };
76
+ }
@@ -165,6 +165,198 @@ export declare const SkillUpdateRequestSchema: z.ZodObject<{
165
165
  scopeIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
166
166
  requestedBy: z.ZodOptional<z.ZodOptional<z.ZodString>>;
167
167
  }, z.core.$strip>;
168
+ /**
169
+ * One dictionary entry for transcription: correct spelling, mis-hearings, meaning.
170
+ */
171
+ export declare const TranscribeDictionaryEntrySchema: z.ZodObject<{
172
+ term: z.ZodString;
173
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
174
+ meaning: z.ZodOptional<z.ZodString>;
175
+ }, z.core.$strip>;
176
+ /**
177
+ * Engine and correction settings shared by `POST /agent/transcribe`, the
178
+ * OpenAI-compatible route's extension fields and the transcribe WebSocket's
179
+ * config frame. Unknown keys (including `credentials`) are stripped: a caller
180
+ * cannot swap in its own key or point an engine at another host.
181
+ */
182
+ export declare const TranscribeConfigSchema: z.ZodObject<{
183
+ provider: z.ZodOptional<z.ZodString>;
184
+ model: z.ZodOptional<z.ZodString>;
185
+ language: z.ZodOptional<z.ZodString>;
186
+ prompt: z.ZodOptional<z.ZodString>;
187
+ dictionary: z.ZodOptional<z.ZodArray<z.ZodObject<{
188
+ term: z.ZodString;
189
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
190
+ meaning: z.ZodOptional<z.ZodString>;
191
+ }, z.core.$strip>>>;
192
+ correction: z.ZodOptional<z.ZodObject<{
193
+ enabled: z.ZodOptional<z.ZodBoolean>;
194
+ guard: z.ZodOptional<z.ZodEnum<{
195
+ none: "none";
196
+ decide: "decide";
197
+ }>>;
198
+ decideTimeoutMs: z.ZodOptional<z.ZodNumber>;
199
+ rewrite: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodObject<{
200
+ provider: z.ZodOptional<z.ZodString>;
201
+ model: z.ZodOptional<z.ZodString>;
202
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
203
+ }, z.core.$strip>]>>;
204
+ transliterate: z.ZodOptional<z.ZodEnum<{
205
+ native: "native";
206
+ latin: "latin";
207
+ }>>;
208
+ punctuate: z.ZodOptional<z.ZodBoolean>;
209
+ secondOpinion: z.ZodOptional<z.ZodObject<{
210
+ provider: z.ZodString;
211
+ model: z.ZodOptional<z.ZodString>;
212
+ }, z.core.$strip>>;
213
+ context: z.ZodOptional<z.ZodString>;
214
+ maxDropRatio: z.ZodOptional<z.ZodNumber>;
215
+ }, z.core.$strip>>;
216
+ fallback: z.ZodOptional<z.ZodObject<{
217
+ provider: z.ZodString;
218
+ model: z.ZodOptional<z.ZodString>;
219
+ when: z.ZodOptional<z.ZodArray<z.ZodEnum<{
220
+ error: "error";
221
+ empty: "empty";
222
+ unsure: "unsure";
223
+ }>>>;
224
+ }, z.core.$strip>>;
225
+ diarization: z.ZodOptional<z.ZodBoolean>;
226
+ wordTimestamps: z.ZodOptional<z.ZodBoolean>;
227
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
228
+ }, z.core.$strip>;
229
+ /**
230
+ * `POST /agent/transcribe` body: the config plus exactly one audio source.
231
+ */
232
+ export declare const TranscribeRequestSchema: z.ZodObject<{
233
+ provider: z.ZodOptional<z.ZodString>;
234
+ model: z.ZodOptional<z.ZodString>;
235
+ language: z.ZodOptional<z.ZodString>;
236
+ prompt: z.ZodOptional<z.ZodString>;
237
+ dictionary: z.ZodOptional<z.ZodArray<z.ZodObject<{
238
+ term: z.ZodString;
239
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
240
+ meaning: z.ZodOptional<z.ZodString>;
241
+ }, z.core.$strip>>>;
242
+ correction: z.ZodOptional<z.ZodObject<{
243
+ enabled: z.ZodOptional<z.ZodBoolean>;
244
+ guard: z.ZodOptional<z.ZodEnum<{
245
+ none: "none";
246
+ decide: "decide";
247
+ }>>;
248
+ decideTimeoutMs: z.ZodOptional<z.ZodNumber>;
249
+ rewrite: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodObject<{
250
+ provider: z.ZodOptional<z.ZodString>;
251
+ model: z.ZodOptional<z.ZodString>;
252
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
253
+ }, z.core.$strip>]>>;
254
+ transliterate: z.ZodOptional<z.ZodEnum<{
255
+ native: "native";
256
+ latin: "latin";
257
+ }>>;
258
+ punctuate: z.ZodOptional<z.ZodBoolean>;
259
+ secondOpinion: z.ZodOptional<z.ZodObject<{
260
+ provider: z.ZodString;
261
+ model: z.ZodOptional<z.ZodString>;
262
+ }, z.core.$strip>>;
263
+ context: z.ZodOptional<z.ZodString>;
264
+ maxDropRatio: z.ZodOptional<z.ZodNumber>;
265
+ }, z.core.$strip>>;
266
+ fallback: z.ZodOptional<z.ZodObject<{
267
+ provider: z.ZodString;
268
+ model: z.ZodOptional<z.ZodString>;
269
+ when: z.ZodOptional<z.ZodArray<z.ZodEnum<{
270
+ error: "error";
271
+ empty: "empty";
272
+ unsure: "unsure";
273
+ }>>>;
274
+ }, z.core.$strip>>;
275
+ diarization: z.ZodOptional<z.ZodBoolean>;
276
+ wordTimestamps: z.ZodOptional<z.ZodBoolean>;
277
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
278
+ audio: z.ZodOptional<z.ZodString>;
279
+ audioUrl: z.ZodOptional<z.ZodString>;
280
+ audioPath: z.ZodOptional<z.ZodString>;
281
+ format: z.ZodOptional<z.ZodEnum<{
282
+ mp3: "mp3";
283
+ wav: "wav";
284
+ ogg: "ogg";
285
+ opus: "opus";
286
+ m4a: "m4a";
287
+ flac: "flac";
288
+ webm: "webm";
289
+ mp4: "mp4";
290
+ mpeg: "mpeg";
291
+ mpga: "mpga";
292
+ pcm16: "pcm16";
293
+ }>>;
294
+ }, z.core.$strip>;
295
+ /**
296
+ * The transcribe WebSocket's first text frame: the config plus streaming tuning.
297
+ */
298
+ export declare const TranscribeStreamConfigSchema: z.ZodObject<{
299
+ provider: z.ZodOptional<z.ZodString>;
300
+ model: z.ZodOptional<z.ZodString>;
301
+ language: z.ZodOptional<z.ZodString>;
302
+ prompt: z.ZodOptional<z.ZodString>;
303
+ dictionary: z.ZodOptional<z.ZodArray<z.ZodObject<{
304
+ term: z.ZodString;
305
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
306
+ meaning: z.ZodOptional<z.ZodString>;
307
+ }, z.core.$strip>>>;
308
+ correction: z.ZodOptional<z.ZodObject<{
309
+ enabled: z.ZodOptional<z.ZodBoolean>;
310
+ guard: z.ZodOptional<z.ZodEnum<{
311
+ none: "none";
312
+ decide: "decide";
313
+ }>>;
314
+ decideTimeoutMs: z.ZodOptional<z.ZodNumber>;
315
+ rewrite: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodObject<{
316
+ provider: z.ZodOptional<z.ZodString>;
317
+ model: z.ZodOptional<z.ZodString>;
318
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
319
+ }, z.core.$strip>]>>;
320
+ transliterate: z.ZodOptional<z.ZodEnum<{
321
+ native: "native";
322
+ latin: "latin";
323
+ }>>;
324
+ punctuate: z.ZodOptional<z.ZodBoolean>;
325
+ secondOpinion: z.ZodOptional<z.ZodObject<{
326
+ provider: z.ZodString;
327
+ model: z.ZodOptional<z.ZodString>;
328
+ }, z.core.$strip>>;
329
+ context: z.ZodOptional<z.ZodString>;
330
+ maxDropRatio: z.ZodOptional<z.ZodNumber>;
331
+ }, z.core.$strip>>;
332
+ fallback: z.ZodOptional<z.ZodObject<{
333
+ provider: z.ZodString;
334
+ model: z.ZodOptional<z.ZodString>;
335
+ when: z.ZodOptional<z.ZodArray<z.ZodEnum<{
336
+ error: "error";
337
+ empty: "empty";
338
+ unsure: "unsure";
339
+ }>>>;
340
+ }, z.core.$strip>>;
341
+ diarization: z.ZodOptional<z.ZodBoolean>;
342
+ wordTimestamps: z.ZodOptional<z.ZodBoolean>;
343
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
344
+ streaming: z.ZodOptional<z.ZodObject<{
345
+ mode: z.ZodOptional<z.ZodEnum<{
346
+ auto: "auto";
347
+ native: "native";
348
+ chunked: "chunked";
349
+ }>>;
350
+ sampleRate: z.ZodOptional<z.ZodNumber>;
351
+ endSilenceMs: z.ZodOptional<z.ZodNumber>;
352
+ intervalMs: z.ZodOptional<z.ZodNumber>;
353
+ softCutSeconds: z.ZodOptional<z.ZodNumber>;
354
+ maxUtteranceSeconds: z.ZodOptional<z.ZodNumber>;
355
+ onsetRms: z.ZodOptional<z.ZodNumber>;
356
+ prerollMs: z.ZodOptional<z.ZodNumber>;
357
+ minSpeechMs: z.ZodOptional<z.ZodNumber>;
358
+ }, z.core.$strip>>;
359
+ }, z.core.$strip>;
168
360
  /**
169
361
  * Type guard to check if a value is an ErrorResponse
170
362
  */
@@ -171,6 +171,106 @@ export const SkillCreateRequestSchema = z.object({
171
171
  * Skill update request schema — all fields optional (patch semantics)
172
172
  */
173
173
  export const SkillUpdateRequestSchema = SkillCreateRequestSchema.partial();
174
+ /**
175
+ * One dictionary entry for transcription: correct spelling, mis-hearings, meaning.
176
+ */
177
+ export const TranscribeDictionaryEntrySchema = z.object({
178
+ term: z.string().trim().min(1, "term is required"),
179
+ heardAs: z.array(z.string()).max(64).optional(),
180
+ meaning: z.string().max(2000).optional(),
181
+ });
182
+ const sttProviderName = z.string().min(1).max(100);
183
+ /**
184
+ * Engine and correction settings shared by `POST /agent/transcribe`, the
185
+ * OpenAI-compatible route's extension fields and the transcribe WebSocket's
186
+ * config frame. Unknown keys (including `credentials`) are stripped: a caller
187
+ * cannot swap in its own key or point an engine at another host.
188
+ */
189
+ export const TranscribeConfigSchema = z.object({
190
+ provider: sttProviderName.optional(),
191
+ model: z.string().min(1).max(200).optional(),
192
+ language: z.string().min(1).max(35).optional(),
193
+ prompt: z.string().max(4000).optional(),
194
+ dictionary: z.array(TranscribeDictionaryEntrySchema).max(500).optional(),
195
+ correction: z
196
+ .object({
197
+ enabled: z.boolean().optional(),
198
+ guard: z.enum(["decide", "none"]).optional(),
199
+ decideTimeoutMs: z.number().int().positive().max(60_000).optional(),
200
+ rewrite: z
201
+ .union([
202
+ z.literal(false),
203
+ z.object({
204
+ provider: z.string().min(1).max(100).optional(),
205
+ model: z.string().min(1).max(200).optional(),
206
+ timeoutMs: z.number().int().positive().max(120_000).optional(),
207
+ }),
208
+ ])
209
+ .optional(),
210
+ transliterate: z.enum(["latin", "native"]).optional(),
211
+ punctuate: z.boolean().optional(),
212
+ secondOpinion: z
213
+ .object({
214
+ provider: sttProviderName,
215
+ model: z.string().min(1).max(200).optional(),
216
+ })
217
+ .optional(),
218
+ context: z.string().max(1000).optional(),
219
+ maxDropRatio: z.number().min(0).max(1).optional(),
220
+ })
221
+ .optional(),
222
+ fallback: z
223
+ .object({
224
+ provider: sttProviderName,
225
+ model: z.string().min(1).max(200).optional(),
226
+ when: z.array(z.enum(["unsure", "empty", "error"])).optional(),
227
+ })
228
+ .optional(),
229
+ diarization: z.boolean().optional(),
230
+ wordTimestamps: z.boolean().optional(),
231
+ timeoutMs: z.number().int().positive().max(600_000).optional(),
232
+ });
233
+ /**
234
+ * `POST /agent/transcribe` body: the config plus exactly one audio source.
235
+ */
236
+ export const TranscribeRequestSchema = TranscribeConfigSchema.extend({
237
+ audio: z.string().min(1).optional(),
238
+ audioUrl: z.string().url().optional(),
239
+ audioPath: z.string().min(1).optional(),
240
+ format: z
241
+ .enum([
242
+ "mp3",
243
+ "wav",
244
+ "ogg",
245
+ "opus",
246
+ "m4a",
247
+ "flac",
248
+ "webm",
249
+ "mp4",
250
+ "mpeg",
251
+ "mpga",
252
+ "pcm16",
253
+ ])
254
+ .optional(),
255
+ }).refine((body) => [body.audio, body.audioUrl, body.audioPath].filter((source) => source !== undefined).length === 1, { message: "Pass exactly one of audio, audioUrl or audioPath" });
256
+ /**
257
+ * The transcribe WebSocket's first text frame: the config plus streaming tuning.
258
+ */
259
+ export const TranscribeStreamConfigSchema = TranscribeConfigSchema.extend({
260
+ streaming: z
261
+ .object({
262
+ mode: z.enum(["auto", "native", "chunked"]).optional(),
263
+ sampleRate: z.number().int().min(8000).max(48000).optional(),
264
+ endSilenceMs: z.number().positive().optional(),
265
+ intervalMs: z.number().positive().optional(),
266
+ softCutSeconds: z.number().positive().optional(),
267
+ maxUtteranceSeconds: z.number().positive().optional(),
268
+ onsetRms: z.number().positive().optional(),
269
+ prerollMs: z.number().nonnegative().optional(),
270
+ minSpeechMs: z.number().nonnegative().optional(),
271
+ })
272
+ .optional(),
273
+ });
174
274
  // ============================================
175
275
  // Error Response Type Guards / Helpers
176
276
  // ============================================
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Transcribe WebSocket — `transcribeStream()` over a socket.
3
+ *
4
+ * Protocol (one transcription per connection):
5
+ * 1. The client's first frame is TEXT: a JSON config with the fields of
6
+ * `POST /agent/transcribe` minus the audio (`provider`, `model`,
7
+ * `language`, `dictionary`, `correction`, `fallback`, `streaming` …).
8
+ * 2. Then BINARY frames of PCM16LE mono audio at `streaming.sampleRate`
9
+ * (default 16000).
10
+ * 3. A TEXT frame `{"type":"end"}` (or closing the socket) ends the audio; the
11
+ * server flushes the last utterance.
12
+ *
13
+ * The server sends one TEXT frame per `TranscribeStreamEvent` (JSON), then
14
+ * closes with 1000. A bad config is answered with an `error` event and close
15
+ * code 1008.
16
+ *
17
+ * The route adapters do not handle upgrades, so this attaches to the Node
18
+ * `http.Server` the caller owns, next to whatever else is listening there; it
19
+ * answers only upgrades on its own path.
20
+ *
21
+ * @module server/websocket/transcribeWebSocket
22
+ */
23
+ import type { Server as HttpServer } from "node:http";
24
+ import type { NeuroLink } from "../../neurolink.js";
25
+ import type { ServerTranscribeWebSocketOptions } from "../../types/index.js";
26
+ /**
27
+ * Serve `transcribeStream()` on `path` (default
28
+ * `/v1/audio/transcriptions/stream`) of an existing HTTP server.
29
+ *
30
+ * @example
31
+ * ```typescript
32
+ * const server = http.createServer(app);
33
+ * const ws = await attachTranscribeWebSocket(server, neurolink, {
34
+ * authToken: process.env.NEUROLINK_SERVER_API_KEY?.split(","),
35
+ * });
36
+ * server.listen(3000);
37
+ * // later: ws.close();
38
+ * ```
39
+ */
40
+ export declare function attachTranscribeWebSocket(server: HttpServer, neurolink: NeuroLink, options?: ServerTranscribeWebSocketOptions): Promise<{
41
+ close: () => void;
42
+ }>;
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Transcribe WebSocket — `transcribeStream()` over a socket.
3
+ *
4
+ * Protocol (one transcription per connection):
5
+ * 1. The client's first frame is TEXT: a JSON config with the fields of
6
+ * `POST /agent/transcribe` minus the audio (`provider`, `model`,
7
+ * `language`, `dictionary`, `correction`, `fallback`, `streaming` …).
8
+ * 2. Then BINARY frames of PCM16LE mono audio at `streaming.sampleRate`
9
+ * (default 16000).
10
+ * 3. A TEXT frame `{"type":"end"}` (or closing the socket) ends the audio; the
11
+ * server flushes the last utterance.
12
+ *
13
+ * The server sends one TEXT frame per `TranscribeStreamEvent` (JSON), then
14
+ * closes with 1000. A bad config is answered with an `error` event and close
15
+ * code 1008.
16
+ *
17
+ * The route adapters do not handle upgrades, so this attaches to the Node
18
+ * `http.Server` the caller owns, next to whatever else is listening there; it
19
+ * answers only upgrades on its own path.
20
+ *
21
+ * @module server/websocket/transcribeWebSocket
22
+ */
23
+ import { logger } from "../../utils/logger.js";
24
+ import { timingSafeEqualString } from "../voice/tokenCompare.js";
25
+ import { TranscribeStreamConfigSchema } from "../utils/validation.js";
26
+ const DEFAULT_PATH = "/v1/audio/transcriptions/stream";
27
+ /** Audio waiting for the engine beyond this (about 8.5 min at 16 kHz) closes the socket. */
28
+ const MAX_QUEUED_BYTES = 16 * 1024 * 1024;
29
+ /** Push-driven async iterable: socket frames in, `transcribeStream()` pulls. */
30
+ class FrameQueue {
31
+ frames = [];
32
+ waiting;
33
+ ended = false;
34
+ queuedBytes = 0;
35
+ push(frame) {
36
+ if (this.ended) {
37
+ return;
38
+ }
39
+ if (this.waiting) {
40
+ const resolve = this.waiting;
41
+ this.waiting = undefined;
42
+ resolve({ value: frame, done: false });
43
+ return;
44
+ }
45
+ this.frames.push(frame);
46
+ this.queuedBytes += frame.length;
47
+ }
48
+ end() {
49
+ this.ended = true;
50
+ if (this.waiting) {
51
+ const resolve = this.waiting;
52
+ this.waiting = undefined;
53
+ resolve({ value: undefined, done: true });
54
+ }
55
+ }
56
+ [Symbol.asyncIterator]() {
57
+ return {
58
+ next: () => {
59
+ const frame = this.frames.shift();
60
+ if (frame) {
61
+ this.queuedBytes -= frame.length;
62
+ return Promise.resolve({ value: frame, done: false });
63
+ }
64
+ if (this.ended) {
65
+ return Promise.resolve({ value: undefined, done: true });
66
+ }
67
+ return new Promise((resolve) => {
68
+ this.waiting = resolve;
69
+ });
70
+ },
71
+ return: () => {
72
+ this.end();
73
+ return Promise.resolve({ value: undefined, done: true });
74
+ },
75
+ };
76
+ }
77
+ }
78
+ /**
79
+ * `true` when the upgrade's `Origin` is on the allow-list, the list allows
80
+ * any origin, or the request carries no `Origin` (not a browser).
81
+ */
82
+ function isOriginAllowed(req, allowed) {
83
+ if (!allowed || allowed.includes("*")) {
84
+ return true;
85
+ }
86
+ const origin = req.headers.origin;
87
+ if (typeof origin !== "string" || origin.length === 0) {
88
+ return true;
89
+ }
90
+ const wanted = origin.trim().toLowerCase();
91
+ return allowed.some((o) => o.trim().toLowerCase() === wanted);
92
+ }
93
+ function isAuthorized(req, token) {
94
+ const tokens = (typeof token === "string" ? [token] : (token ?? [])).filter((t) => t.length > 0);
95
+ if (tokens.length === 0) {
96
+ return true;
97
+ }
98
+ const header = req.headers.authorization;
99
+ const headerToken = typeof header === "string" && header.startsWith("Bearer ")
100
+ ? header.slice(7)
101
+ : undefined;
102
+ let urlToken;
103
+ try {
104
+ urlToken =
105
+ new URL(req.url ?? "/", "http://localhost").searchParams.get("token") ??
106
+ undefined;
107
+ }
108
+ catch {
109
+ urlToken = undefined;
110
+ }
111
+ const provided = headerToken ?? urlToken;
112
+ return (provided !== undefined &&
113
+ tokens.some((t) => timingSafeEqualString(provided, t)));
114
+ }
115
+ function pathOf(req) {
116
+ try {
117
+ return new URL(req.url ?? "/", "http://localhost").pathname;
118
+ }
119
+ catch {
120
+ return "";
121
+ }
122
+ }
123
+ /**
124
+ * Serve `transcribeStream()` on `path` (default
125
+ * `/v1/audio/transcriptions/stream`) of an existing HTTP server.
126
+ *
127
+ * @example
128
+ * ```typescript
129
+ * const server = http.createServer(app);
130
+ * const ws = await attachTranscribeWebSocket(server, neurolink, {
131
+ * authToken: process.env.NEUROLINK_SERVER_API_KEY?.split(","),
132
+ * });
133
+ * server.listen(3000);
134
+ * // later: ws.close();
135
+ * ```
136
+ */
137
+ export async function attachTranscribeWebSocket(server, neurolink, options = {}) {
138
+ // Loaded on use so `ws` stays off the package's static import graph.
139
+ const { WebSocketServer, WebSocket } = await import("ws");
140
+ const path = options.path ?? DEFAULT_PATH;
141
+ const wss = new WebSocketServer({
142
+ noServer: true,
143
+ maxPayload: options.maxPayload ?? 1_048_576,
144
+ });
145
+ const onUpgrade = (req, socket, head) => {
146
+ if (pathOf(req) !== path) {
147
+ return; // Someone else's upgrade.
148
+ }
149
+ if (!isOriginAllowed(req, options.allowedOrigins)) {
150
+ socket.write("HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n");
151
+ socket.destroy();
152
+ return;
153
+ }
154
+ if (!isAuthorized(req, options.authToken)) {
155
+ socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
156
+ socket.destroy();
157
+ return;
158
+ }
159
+ wss.handleUpgrade(req, socket, head, (client) => {
160
+ wss.emit("connection", client, req);
161
+ });
162
+ };
163
+ server.on("upgrade", onUpgrade);
164
+ wss.on("connection", (client) => {
165
+ const queue = new FrameQueue();
166
+ let started = false;
167
+ const send = (event) => {
168
+ if (client.readyState !== WebSocket.OPEN) {
169
+ return false;
170
+ }
171
+ client.send(JSON.stringify(event));
172
+ return true;
173
+ };
174
+ const run = async (config) => {
175
+ try {
176
+ for await (const event of neurolink.transcribeStream({
177
+ ...config,
178
+ audio: queue,
179
+ })) {
180
+ if (!send(event)) {
181
+ break; // The client is gone; stop pulling.
182
+ }
183
+ }
184
+ if (client.readyState === WebSocket.OPEN) {
185
+ client.close(1000, "done");
186
+ }
187
+ }
188
+ catch (error) {
189
+ const message = error instanceof Error ? error.message : String(error);
190
+ logger.warn("[TranscribeWebSocket] stream failed", { error: message });
191
+ send({ type: "error", message, recoverable: false });
192
+ if (client.readyState === WebSocket.OPEN) {
193
+ client.close(1011, "transcription failed");
194
+ }
195
+ }
196
+ };
197
+ client.on("message", (data, isBinary) => {
198
+ const frame = Buffer.isBuffer(data)
199
+ ? data
200
+ : Array.isArray(data)
201
+ ? Buffer.concat(data)
202
+ : Buffer.from(data);
203
+ if (!started) {
204
+ if (isBinary) {
205
+ send({
206
+ type: "error",
207
+ message: "The first frame must be a JSON config (text), before any audio",
208
+ recoverable: false,
209
+ });
210
+ client.close(1008, "config first");
211
+ return;
212
+ }
213
+ let parsed;
214
+ try {
215
+ parsed = JSON.parse(frame.toString("utf-8"));
216
+ }
217
+ catch {
218
+ parsed = undefined;
219
+ }
220
+ const config = TranscribeStreamConfigSchema.safeParse(parsed);
221
+ if (!config.success) {
222
+ send({
223
+ type: "error",
224
+ message: `Invalid config: ${config.error.issues
225
+ .map((issue) => `${issue.path.join(".") || "config"} ${issue.message}`)
226
+ .join("; ")}`,
227
+ recoverable: false,
228
+ });
229
+ client.close(1008, "invalid config");
230
+ return;
231
+ }
232
+ started = true;
233
+ void run(config.data);
234
+ return;
235
+ }
236
+ if (isBinary) {
237
+ queue.push(frame);
238
+ if (queue.queuedBytes > MAX_QUEUED_BYTES) {
239
+ send({
240
+ type: "error",
241
+ message: "Audio is arriving faster than it can be transcribed",
242
+ recoverable: false,
243
+ });
244
+ queue.end();
245
+ client.close(1009, "backlog");
246
+ }
247
+ return;
248
+ }
249
+ // Any text frame after the config ends the audio ({"type":"end"}).
250
+ queue.end();
251
+ });
252
+ client.on("close", () => queue.end());
253
+ client.on("error", () => queue.end());
254
+ });
255
+ return {
256
+ close: () => {
257
+ server.off("upgrade", onUpgrade);
258
+ for (const client of wss.clients) {
259
+ client.terminate();
260
+ }
261
+ wss.close();
262
+ },
263
+ };
264
+ }