@juspay/neurolink 12.48.2 → 12.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +3 -3
  2. package/dist/browser/neurolink.min.js +616 -595
  3. package/dist/cli/commands/serve.d.ts +0 -3
  4. package/dist/cli/commands/serve.js +85 -3
  5. package/dist/cli/commands/transcribe.d.ts +24 -0
  6. package/dist/cli/commands/transcribe.js +718 -0
  7. package/dist/cli/factories/commandFactory.d.ts +10 -0
  8. package/dist/cli/factories/commandFactory.js +13 -0
  9. package/dist/cli/parser.js +3 -0
  10. package/dist/factories/mediaHandlerCatalog.js +11 -9
  11. package/dist/factories/sttDescriptors.d.ts +63 -0
  12. package/dist/factories/sttDescriptors.js +286 -0
  13. package/dist/index.d.ts +4 -1
  14. package/dist/index.js +7 -1
  15. package/dist/neurolink.d.ts +68 -1
  16. package/dist/neurolink.js +202 -4
  17. package/dist/processors/media/AudioProcessor.d.ts +15 -56
  18. package/dist/processors/media/AudioProcessor.js +105 -276
  19. package/dist/processors/media/VideoProcessor.d.ts +3 -5
  20. package/dist/processors/media/VideoProcessor.js +52 -51
  21. package/dist/server/abstract/baseServerAdapter.d.ts +7 -0
  22. package/dist/server/abstract/baseServerAdapter.js +13 -1
  23. package/dist/server/adapters/expressAdapter.d.ts +1 -0
  24. package/dist/server/adapters/expressAdapter.js +48 -4
  25. package/dist/server/adapters/fastifyAdapter.d.ts +1 -0
  26. package/dist/server/adapters/fastifyAdapter.js +38 -1
  27. package/dist/server/adapters/honoAdapter.d.ts +1 -4
  28. package/dist/server/adapters/honoAdapter.js +83 -1
  29. package/dist/server/adapters/koaAdapter.d.ts +1 -0
  30. package/dist/server/adapters/koaAdapter.js +43 -0
  31. package/dist/server/index.d.ts +3 -2
  32. package/dist/server/index.js +3 -2
  33. package/dist/server/routes/index.d.ts +1 -0
  34. package/dist/server/routes/index.js +3 -0
  35. package/dist/server/routes/transcribeRoutes.d.ts +23 -0
  36. package/dist/server/routes/transcribeRoutes.js +506 -0
  37. package/dist/server/utils/rawBody.d.ts +32 -0
  38. package/dist/server/utils/rawBody.js +76 -0
  39. package/dist/server/utils/validation.d.ts +192 -0
  40. package/dist/server/utils/validation.js +100 -0
  41. package/dist/server/websocket/transcribeWebSocket.d.ts +42 -0
  42. package/dist/server/websocket/transcribeWebSocket.js +264 -0
  43. package/dist/types/cli.d.ts +46 -0
  44. package/dist/types/index.d.ts +1 -0
  45. package/dist/types/index.js +1 -0
  46. package/dist/types/providers.d.ts +7 -1
  47. package/dist/types/server.d.ts +117 -1
  48. package/dist/types/stt.d.ts +160 -7
  49. package/dist/types/transcribe.d.ts +506 -0
  50. package/dist/types/transcribe.js +29 -0
  51. package/dist/utils/speakerTurns.d.ts +28 -0
  52. package/dist/utils/speakerTurns.js +93 -0
  53. package/dist/utils/sttProcessor.d.ts +20 -2
  54. package/dist/utils/sttProcessor.js +73 -5
  55. package/dist/voice/correction/dictionary.d.ts +49 -0
  56. package/dist/voice/correction/dictionary.js +134 -0
  57. package/dist/voice/correction/guard.d.ts +35 -0
  58. package/dist/voice/correction/guard.js +161 -0
  59. package/dist/voice/correction/index.d.ts +47 -0
  60. package/dist/voice/correction/index.js +211 -0
  61. package/dist/voice/correction/rewrite.d.ts +92 -0
  62. package/dist/voice/correction/rewrite.js +322 -0
  63. package/dist/voice/errors.d.ts +15 -0
  64. package/dist/voice/errors.js +45 -0
  65. package/dist/voice/index.d.ts +20 -2
  66. package/dist/voice/index.js +75 -9
  67. package/dist/voice/providers/AzureSTT.d.ts +10 -7
  68. package/dist/voice/providers/AzureSTT.js +23 -75
  69. package/dist/voice/providers/DeepgramSTT.d.ts +17 -2
  70. package/dist/voice/providers/DeepgramSTT.js +109 -13
  71. package/dist/voice/providers/ElevenLabsSTT.d.ts +12 -2
  72. package/dist/voice/providers/ElevenLabsSTT.js +34 -8
  73. package/dist/voice/providers/GoogleSTT.d.ts +10 -8
  74. package/dist/voice/providers/GoogleSTT.js +21 -75
  75. package/dist/voice/providers/OpenAISTT.d.ts +30 -9
  76. package/dist/voice/providers/OpenAISTT.js +196 -21
  77. package/dist/voice/providers/WhistleSTT.d.ts +87 -0
  78. package/dist/voice/providers/WhistleSTT.js +313 -0
  79. package/dist/voice/streaming/chunkedStreamAdapter.d.ts +19 -0
  80. package/dist/voice/streaming/chunkedStreamAdapter.js +519 -0
  81. package/dist/voice/streaming/endpointer.d.ts +81 -0
  82. package/dist/voice/streaming/endpointer.js +247 -0
  83. package/dist/voice/streaming/localAgreement.d.ts +30 -0
  84. package/dist/voice/streaming/localAgreement.js +44 -0
  85. package/dist/voice/sttEnvCredentials.d.ts +12 -0
  86. package/dist/voice/sttEnvCredentials.js +12 -0
  87. package/dist/voice/transcribe.d.ts +47 -0
  88. package/dist/voice/transcribe.js +519 -0
  89. package/dist/voice/transcribeStream.d.ts +4 -0
  90. package/dist/voice/transcribeStream.js +584 -0
  91. package/dist/voice/whistle/assets.d.ts +47 -0
  92. package/dist/voice/whistle/assets.js +270 -0
  93. package/dist/voice/whistle/audio.d.ts +32 -0
  94. package/dist/voice/whistle/audio.js +267 -0
  95. package/dist/voice/whistle/engine.d.ts +32 -0
  96. package/dist/voice/whistle/engine.js +193 -0
  97. package/dist/voice/whistle/worker.d.ts +18 -0
  98. package/dist/voice/whistle/worker.js +294 -0
  99. package/docs-site/static/search-index.json +66 -8
  100. package/models/whistle/README.md +18 -0
  101. package/models/whistle/needle.js +2 -0
  102. package/models/whistle/needle.wasm +0 -0
  103. package/models/whistle/whistle.cact +0 -0
  104. package/package.json +7 -3
@@ -165,6 +165,198 @@ export declare const SkillUpdateRequestSchema: z.ZodObject<{
165
165
  scopeIds: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodString>>>;
166
166
  requestedBy: z.ZodOptional<z.ZodOptional<z.ZodString>>;
167
167
  }, z.core.$strip>;
168
+ /**
169
+ * One dictionary entry for transcription: correct spelling, mis-hearings, meaning.
170
+ */
171
+ export declare const TranscribeDictionaryEntrySchema: z.ZodObject<{
172
+ term: z.ZodString;
173
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
174
+ meaning: z.ZodOptional<z.ZodString>;
175
+ }, z.core.$strip>;
176
+ /**
177
+ * Engine and correction settings shared by `POST /agent/transcribe`, the
178
+ * OpenAI-compatible route's extension fields and the transcribe WebSocket's
179
+ * config frame. Unknown keys (including `credentials`) are stripped: a caller
180
+ * cannot swap in its own key or point an engine at another host.
181
+ */
182
+ export declare const TranscribeConfigSchema: z.ZodObject<{
183
+ provider: z.ZodOptional<z.ZodString>;
184
+ model: z.ZodOptional<z.ZodString>;
185
+ language: z.ZodOptional<z.ZodString>;
186
+ prompt: z.ZodOptional<z.ZodString>;
187
+ dictionary: z.ZodOptional<z.ZodArray<z.ZodObject<{
188
+ term: z.ZodString;
189
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
190
+ meaning: z.ZodOptional<z.ZodString>;
191
+ }, z.core.$strip>>>;
192
+ correction: z.ZodOptional<z.ZodObject<{
193
+ enabled: z.ZodOptional<z.ZodBoolean>;
194
+ guard: z.ZodOptional<z.ZodEnum<{
195
+ none: "none";
196
+ decide: "decide";
197
+ }>>;
198
+ decideTimeoutMs: z.ZodOptional<z.ZodNumber>;
199
+ rewrite: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodObject<{
200
+ provider: z.ZodOptional<z.ZodString>;
201
+ model: z.ZodOptional<z.ZodString>;
202
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
203
+ }, z.core.$strip>]>>;
204
+ transliterate: z.ZodOptional<z.ZodEnum<{
205
+ native: "native";
206
+ latin: "latin";
207
+ }>>;
208
+ punctuate: z.ZodOptional<z.ZodBoolean>;
209
+ secondOpinion: z.ZodOptional<z.ZodObject<{
210
+ provider: z.ZodString;
211
+ model: z.ZodOptional<z.ZodString>;
212
+ }, z.core.$strip>>;
213
+ context: z.ZodOptional<z.ZodString>;
214
+ maxDropRatio: z.ZodOptional<z.ZodNumber>;
215
+ }, z.core.$strip>>;
216
+ fallback: z.ZodOptional<z.ZodObject<{
217
+ provider: z.ZodString;
218
+ model: z.ZodOptional<z.ZodString>;
219
+ when: z.ZodOptional<z.ZodArray<z.ZodEnum<{
220
+ error: "error";
221
+ empty: "empty";
222
+ unsure: "unsure";
223
+ }>>>;
224
+ }, z.core.$strip>>;
225
+ diarization: z.ZodOptional<z.ZodBoolean>;
226
+ wordTimestamps: z.ZodOptional<z.ZodBoolean>;
227
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
228
+ }, z.core.$strip>;
229
+ /**
230
+ * `POST /agent/transcribe` body: the config plus exactly one audio source.
231
+ */
232
+ export declare const TranscribeRequestSchema: z.ZodObject<{
233
+ provider: z.ZodOptional<z.ZodString>;
234
+ model: z.ZodOptional<z.ZodString>;
235
+ language: z.ZodOptional<z.ZodString>;
236
+ prompt: z.ZodOptional<z.ZodString>;
237
+ dictionary: z.ZodOptional<z.ZodArray<z.ZodObject<{
238
+ term: z.ZodString;
239
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
240
+ meaning: z.ZodOptional<z.ZodString>;
241
+ }, z.core.$strip>>>;
242
+ correction: z.ZodOptional<z.ZodObject<{
243
+ enabled: z.ZodOptional<z.ZodBoolean>;
244
+ guard: z.ZodOptional<z.ZodEnum<{
245
+ none: "none";
246
+ decide: "decide";
247
+ }>>;
248
+ decideTimeoutMs: z.ZodOptional<z.ZodNumber>;
249
+ rewrite: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodObject<{
250
+ provider: z.ZodOptional<z.ZodString>;
251
+ model: z.ZodOptional<z.ZodString>;
252
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
253
+ }, z.core.$strip>]>>;
254
+ transliterate: z.ZodOptional<z.ZodEnum<{
255
+ native: "native";
256
+ latin: "latin";
257
+ }>>;
258
+ punctuate: z.ZodOptional<z.ZodBoolean>;
259
+ secondOpinion: z.ZodOptional<z.ZodObject<{
260
+ provider: z.ZodString;
261
+ model: z.ZodOptional<z.ZodString>;
262
+ }, z.core.$strip>>;
263
+ context: z.ZodOptional<z.ZodString>;
264
+ maxDropRatio: z.ZodOptional<z.ZodNumber>;
265
+ }, z.core.$strip>>;
266
+ fallback: z.ZodOptional<z.ZodObject<{
267
+ provider: z.ZodString;
268
+ model: z.ZodOptional<z.ZodString>;
269
+ when: z.ZodOptional<z.ZodArray<z.ZodEnum<{
270
+ error: "error";
271
+ empty: "empty";
272
+ unsure: "unsure";
273
+ }>>>;
274
+ }, z.core.$strip>>;
275
+ diarization: z.ZodOptional<z.ZodBoolean>;
276
+ wordTimestamps: z.ZodOptional<z.ZodBoolean>;
277
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
278
+ audio: z.ZodOptional<z.ZodString>;
279
+ audioUrl: z.ZodOptional<z.ZodString>;
280
+ audioPath: z.ZodOptional<z.ZodString>;
281
+ format: z.ZodOptional<z.ZodEnum<{
282
+ mp3: "mp3";
283
+ wav: "wav";
284
+ ogg: "ogg";
285
+ opus: "opus";
286
+ m4a: "m4a";
287
+ flac: "flac";
288
+ webm: "webm";
289
+ mp4: "mp4";
290
+ mpeg: "mpeg";
291
+ mpga: "mpga";
292
+ pcm16: "pcm16";
293
+ }>>;
294
+ }, z.core.$strip>;
295
+ /**
296
+ * The transcribe WebSocket's first text frame: the config plus streaming tuning.
297
+ */
298
+ export declare const TranscribeStreamConfigSchema: z.ZodObject<{
299
+ provider: z.ZodOptional<z.ZodString>;
300
+ model: z.ZodOptional<z.ZodString>;
301
+ language: z.ZodOptional<z.ZodString>;
302
+ prompt: z.ZodOptional<z.ZodString>;
303
+ dictionary: z.ZodOptional<z.ZodArray<z.ZodObject<{
304
+ term: z.ZodString;
305
+ heardAs: z.ZodOptional<z.ZodArray<z.ZodString>>;
306
+ meaning: z.ZodOptional<z.ZodString>;
307
+ }, z.core.$strip>>>;
308
+ correction: z.ZodOptional<z.ZodObject<{
309
+ enabled: z.ZodOptional<z.ZodBoolean>;
310
+ guard: z.ZodOptional<z.ZodEnum<{
311
+ none: "none";
312
+ decide: "decide";
313
+ }>>;
314
+ decideTimeoutMs: z.ZodOptional<z.ZodNumber>;
315
+ rewrite: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodObject<{
316
+ provider: z.ZodOptional<z.ZodString>;
317
+ model: z.ZodOptional<z.ZodString>;
318
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
319
+ }, z.core.$strip>]>>;
320
+ transliterate: z.ZodOptional<z.ZodEnum<{
321
+ native: "native";
322
+ latin: "latin";
323
+ }>>;
324
+ punctuate: z.ZodOptional<z.ZodBoolean>;
325
+ secondOpinion: z.ZodOptional<z.ZodObject<{
326
+ provider: z.ZodString;
327
+ model: z.ZodOptional<z.ZodString>;
328
+ }, z.core.$strip>>;
329
+ context: z.ZodOptional<z.ZodString>;
330
+ maxDropRatio: z.ZodOptional<z.ZodNumber>;
331
+ }, z.core.$strip>>;
332
+ fallback: z.ZodOptional<z.ZodObject<{
333
+ provider: z.ZodString;
334
+ model: z.ZodOptional<z.ZodString>;
335
+ when: z.ZodOptional<z.ZodArray<z.ZodEnum<{
336
+ error: "error";
337
+ empty: "empty";
338
+ unsure: "unsure";
339
+ }>>>;
340
+ }, z.core.$strip>>;
341
+ diarization: z.ZodOptional<z.ZodBoolean>;
342
+ wordTimestamps: z.ZodOptional<z.ZodBoolean>;
343
+ timeoutMs: z.ZodOptional<z.ZodNumber>;
344
+ streaming: z.ZodOptional<z.ZodObject<{
345
+ mode: z.ZodOptional<z.ZodEnum<{
346
+ auto: "auto";
347
+ native: "native";
348
+ chunked: "chunked";
349
+ }>>;
350
+ sampleRate: z.ZodOptional<z.ZodNumber>;
351
+ endSilenceMs: z.ZodOptional<z.ZodNumber>;
352
+ intervalMs: z.ZodOptional<z.ZodNumber>;
353
+ softCutSeconds: z.ZodOptional<z.ZodNumber>;
354
+ maxUtteranceSeconds: z.ZodOptional<z.ZodNumber>;
355
+ onsetRms: z.ZodOptional<z.ZodNumber>;
356
+ prerollMs: z.ZodOptional<z.ZodNumber>;
357
+ minSpeechMs: z.ZodOptional<z.ZodNumber>;
358
+ }, z.core.$strip>>;
359
+ }, z.core.$strip>;
168
360
  /**
169
361
  * Type guard to check if a value is an ErrorResponse
170
362
  */
@@ -171,6 +171,106 @@ export const SkillCreateRequestSchema = z.object({
171
171
  * Skill update request schema — all fields optional (patch semantics)
172
172
  */
173
173
  export const SkillUpdateRequestSchema = SkillCreateRequestSchema.partial();
174
+ /**
175
+ * One dictionary entry for transcription: correct spelling, mis-hearings, meaning.
176
+ */
177
+ export const TranscribeDictionaryEntrySchema = z.object({
178
+ term: z.string().trim().min(1, "term is required"),
179
+ heardAs: z.array(z.string()).max(64).optional(),
180
+ meaning: z.string().max(2000).optional(),
181
+ });
182
+ const sttProviderName = z.string().min(1).max(100);
183
+ /**
184
+ * Engine and correction settings shared by `POST /agent/transcribe`, the
185
+ * OpenAI-compatible route's extension fields and the transcribe WebSocket's
186
+ * config frame. Unknown keys (including `credentials`) are stripped: a caller
187
+ * cannot swap in its own key or point an engine at another host.
188
+ */
189
+ export const TranscribeConfigSchema = z.object({
190
+ provider: sttProviderName.optional(),
191
+ model: z.string().min(1).max(200).optional(),
192
+ language: z.string().min(1).max(35).optional(),
193
+ prompt: z.string().max(4000).optional(),
194
+ dictionary: z.array(TranscribeDictionaryEntrySchema).max(500).optional(),
195
+ correction: z
196
+ .object({
197
+ enabled: z.boolean().optional(),
198
+ guard: z.enum(["decide", "none"]).optional(),
199
+ decideTimeoutMs: z.number().int().positive().max(60_000).optional(),
200
+ rewrite: z
201
+ .union([
202
+ z.literal(false),
203
+ z.object({
204
+ provider: z.string().min(1).max(100).optional(),
205
+ model: z.string().min(1).max(200).optional(),
206
+ timeoutMs: z.number().int().positive().max(120_000).optional(),
207
+ }),
208
+ ])
209
+ .optional(),
210
+ transliterate: z.enum(["latin", "native"]).optional(),
211
+ punctuate: z.boolean().optional(),
212
+ secondOpinion: z
213
+ .object({
214
+ provider: sttProviderName,
215
+ model: z.string().min(1).max(200).optional(),
216
+ })
217
+ .optional(),
218
+ context: z.string().max(1000).optional(),
219
+ maxDropRatio: z.number().min(0).max(1).optional(),
220
+ })
221
+ .optional(),
222
+ fallback: z
223
+ .object({
224
+ provider: sttProviderName,
225
+ model: z.string().min(1).max(200).optional(),
226
+ when: z.array(z.enum(["unsure", "empty", "error"])).optional(),
227
+ })
228
+ .optional(),
229
+ diarization: z.boolean().optional(),
230
+ wordTimestamps: z.boolean().optional(),
231
+ timeoutMs: z.number().int().positive().max(600_000).optional(),
232
+ });
233
+ /**
234
+ * `POST /agent/transcribe` body: the config plus exactly one audio source.
235
+ */
236
+ export const TranscribeRequestSchema = TranscribeConfigSchema.extend({
237
+ audio: z.string().min(1).optional(),
238
+ audioUrl: z.string().url().optional(),
239
+ audioPath: z.string().min(1).optional(),
240
+ format: z
241
+ .enum([
242
+ "mp3",
243
+ "wav",
244
+ "ogg",
245
+ "opus",
246
+ "m4a",
247
+ "flac",
248
+ "webm",
249
+ "mp4",
250
+ "mpeg",
251
+ "mpga",
252
+ "pcm16",
253
+ ])
254
+ .optional(),
255
+ }).refine((body) => [body.audio, body.audioUrl, body.audioPath].filter((source) => source !== undefined).length === 1, { message: "Pass exactly one of audio, audioUrl or audioPath" });
256
+ /**
257
+ * The transcribe WebSocket's first text frame: the config plus streaming tuning.
258
+ */
259
+ export const TranscribeStreamConfigSchema = TranscribeConfigSchema.extend({
260
+ streaming: z
261
+ .object({
262
+ mode: z.enum(["auto", "native", "chunked"]).optional(),
263
+ sampleRate: z.number().int().min(8000).max(48000).optional(),
264
+ endSilenceMs: z.number().positive().optional(),
265
+ intervalMs: z.number().positive().optional(),
266
+ softCutSeconds: z.number().positive().optional(),
267
+ maxUtteranceSeconds: z.number().positive().optional(),
268
+ onsetRms: z.number().positive().optional(),
269
+ prerollMs: z.number().nonnegative().optional(),
270
+ minSpeechMs: z.number().nonnegative().optional(),
271
+ })
272
+ .optional(),
273
+ });
174
274
  // ============================================
175
275
  // Error Response Type Guards / Helpers
176
276
  // ============================================
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Transcribe WebSocket — `transcribeStream()` over a socket.
3
+ *
4
+ * Protocol (one transcription per connection):
5
+ * 1. The client's first frame is TEXT: a JSON config with the fields of
6
+ * `POST /agent/transcribe` minus the audio (`provider`, `model`,
7
+ * `language`, `dictionary`, `correction`, `fallback`, `streaming` …).
8
+ * 2. Then BINARY frames of PCM16LE mono audio at `streaming.sampleRate`
9
+ * (default 16000).
10
+ * 3. A TEXT frame `{"type":"end"}` (or closing the socket) ends the audio; the
11
+ * server flushes the last utterance.
12
+ *
13
+ * The server sends one TEXT frame per `TranscribeStreamEvent` (JSON), then
14
+ * closes with 1000. A bad config is answered with an `error` event and close
15
+ * code 1008.
16
+ *
17
+ * The route adapters do not handle upgrades, so this attaches to the Node
18
+ * `http.Server` the caller owns, next to whatever else is listening there; it
19
+ * answers only upgrades on its own path.
20
+ *
21
+ * @module server/websocket/transcribeWebSocket
22
+ */
23
+ import type { Server as HttpServer } from "node:http";
24
+ import type { NeuroLink } from "../../neurolink.js";
25
+ import type { ServerTranscribeWebSocketOptions } from "../../types/index.js";
26
+ /**
27
+ * Serve `transcribeStream()` on `path` (default
28
+ * `/v1/audio/transcriptions/stream`) of an existing HTTP server.
29
+ *
30
+ * @example
31
+ * ```typescript
32
+ * const server = http.createServer(app);
33
+ * const ws = await attachTranscribeWebSocket(server, neurolink, {
34
+ * authToken: process.env.NEUROLINK_SERVER_API_KEY?.split(","),
35
+ * });
36
+ * server.listen(3000);
37
+ * // later: ws.close();
38
+ * ```
39
+ */
40
+ export declare function attachTranscribeWebSocket(server: HttpServer, neurolink: NeuroLink, options?: ServerTranscribeWebSocketOptions): Promise<{
41
+ close: () => void;
42
+ }>;
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Transcribe WebSocket — `transcribeStream()` over a socket.
3
+ *
4
+ * Protocol (one transcription per connection):
5
+ * 1. The client's first frame is TEXT: a JSON config with the fields of
6
+ * `POST /agent/transcribe` minus the audio (`provider`, `model`,
7
+ * `language`, `dictionary`, `correction`, `fallback`, `streaming` …).
8
+ * 2. Then BINARY frames of PCM16LE mono audio at `streaming.sampleRate`
9
+ * (default 16000).
10
+ * 3. A TEXT frame `{"type":"end"}` (or closing the socket) ends the audio; the
11
+ * server flushes the last utterance.
12
+ *
13
+ * The server sends one TEXT frame per `TranscribeStreamEvent` (JSON), then
14
+ * closes with 1000. A bad config is answered with an `error` event and close
15
+ * code 1008.
16
+ *
17
+ * The route adapters do not handle upgrades, so this attaches to the Node
18
+ * `http.Server` the caller owns, next to whatever else is listening there; it
19
+ * answers only upgrades on its own path.
20
+ *
21
+ * @module server/websocket/transcribeWebSocket
22
+ */
23
+ import { logger } from "../../utils/logger.js";
24
+ import { timingSafeEqualString } from "../voice/tokenCompare.js";
25
+ import { TranscribeStreamConfigSchema } from "../utils/validation.js";
26
+ const DEFAULT_PATH = "/v1/audio/transcriptions/stream";
27
+ /** Audio waiting for the engine beyond this (about 8.5 min at 16 kHz) closes the socket. */
28
+ const MAX_QUEUED_BYTES = 16 * 1024 * 1024;
29
+ /** Push-driven async iterable: socket frames in, `transcribeStream()` pulls. */
30
+ class FrameQueue {
31
+ frames = [];
32
+ waiting;
33
+ ended = false;
34
+ queuedBytes = 0;
35
+ push(frame) {
36
+ if (this.ended) {
37
+ return;
38
+ }
39
+ if (this.waiting) {
40
+ const resolve = this.waiting;
41
+ this.waiting = undefined;
42
+ resolve({ value: frame, done: false });
43
+ return;
44
+ }
45
+ this.frames.push(frame);
46
+ this.queuedBytes += frame.length;
47
+ }
48
+ end() {
49
+ this.ended = true;
50
+ if (this.waiting) {
51
+ const resolve = this.waiting;
52
+ this.waiting = undefined;
53
+ resolve({ value: undefined, done: true });
54
+ }
55
+ }
56
+ [Symbol.asyncIterator]() {
57
+ return {
58
+ next: () => {
59
+ const frame = this.frames.shift();
60
+ if (frame) {
61
+ this.queuedBytes -= frame.length;
62
+ return Promise.resolve({ value: frame, done: false });
63
+ }
64
+ if (this.ended) {
65
+ return Promise.resolve({ value: undefined, done: true });
66
+ }
67
+ return new Promise((resolve) => {
68
+ this.waiting = resolve;
69
+ });
70
+ },
71
+ return: () => {
72
+ this.end();
73
+ return Promise.resolve({ value: undefined, done: true });
74
+ },
75
+ };
76
+ }
77
+ }
78
+ /**
79
+ * `true` when the upgrade's `Origin` is on the allow-list, the list allows
80
+ * any origin, or the request carries no `Origin` (not a browser).
81
+ */
82
+ function isOriginAllowed(req, allowed) {
83
+ if (!allowed || allowed.includes("*")) {
84
+ return true;
85
+ }
86
+ const origin = req.headers.origin;
87
+ if (typeof origin !== "string" || origin.length === 0) {
88
+ return true;
89
+ }
90
+ const wanted = origin.trim().toLowerCase();
91
+ return allowed.some((o) => o.trim().toLowerCase() === wanted);
92
+ }
93
+ function isAuthorized(req, token) {
94
+ const tokens = (typeof token === "string" ? [token] : (token ?? [])).filter((t) => t.length > 0);
95
+ if (tokens.length === 0) {
96
+ return true;
97
+ }
98
+ const header = req.headers.authorization;
99
+ const headerToken = typeof header === "string" && header.startsWith("Bearer ")
100
+ ? header.slice(7)
101
+ : undefined;
102
+ let urlToken;
103
+ try {
104
+ urlToken =
105
+ new URL(req.url ?? "/", "http://localhost").searchParams.get("token") ??
106
+ undefined;
107
+ }
108
+ catch {
109
+ urlToken = undefined;
110
+ }
111
+ const provided = headerToken ?? urlToken;
112
+ return (provided !== undefined &&
113
+ tokens.some((t) => timingSafeEqualString(provided, t)));
114
+ }
115
+ function pathOf(req) {
116
+ try {
117
+ return new URL(req.url ?? "/", "http://localhost").pathname;
118
+ }
119
+ catch {
120
+ return "";
121
+ }
122
+ }
123
+ /**
124
+ * Serve `transcribeStream()` on `path` (default
125
+ * `/v1/audio/transcriptions/stream`) of an existing HTTP server.
126
+ *
127
+ * @example
128
+ * ```typescript
129
+ * const server = http.createServer(app);
130
+ * const ws = await attachTranscribeWebSocket(server, neurolink, {
131
+ * authToken: process.env.NEUROLINK_SERVER_API_KEY?.split(","),
132
+ * });
133
+ * server.listen(3000);
134
+ * // later: ws.close();
135
+ * ```
136
+ */
137
+ export async function attachTranscribeWebSocket(server, neurolink, options = {}) {
138
+ // Loaded on use so `ws` stays off the package's static import graph.
139
+ const { WebSocketServer, WebSocket } = await import("ws");
140
+ const path = options.path ?? DEFAULT_PATH;
141
+ const wss = new WebSocketServer({
142
+ noServer: true,
143
+ maxPayload: options.maxPayload ?? 1_048_576,
144
+ });
145
+ const onUpgrade = (req, socket, head) => {
146
+ if (pathOf(req) !== path) {
147
+ return; // Someone else's upgrade.
148
+ }
149
+ if (!isOriginAllowed(req, options.allowedOrigins)) {
150
+ socket.write("HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n");
151
+ socket.destroy();
152
+ return;
153
+ }
154
+ if (!isAuthorized(req, options.authToken)) {
155
+ socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
156
+ socket.destroy();
157
+ return;
158
+ }
159
+ wss.handleUpgrade(req, socket, head, (client) => {
160
+ wss.emit("connection", client, req);
161
+ });
162
+ };
163
+ server.on("upgrade", onUpgrade);
164
+ wss.on("connection", (client) => {
165
+ const queue = new FrameQueue();
166
+ let started = false;
167
+ const send = (event) => {
168
+ if (client.readyState !== WebSocket.OPEN) {
169
+ return false;
170
+ }
171
+ client.send(JSON.stringify(event));
172
+ return true;
173
+ };
174
+ const run = async (config) => {
175
+ try {
176
+ for await (const event of neurolink.transcribeStream({
177
+ ...config,
178
+ audio: queue,
179
+ })) {
180
+ if (!send(event)) {
181
+ break; // The client is gone; stop pulling.
182
+ }
183
+ }
184
+ if (client.readyState === WebSocket.OPEN) {
185
+ client.close(1000, "done");
186
+ }
187
+ }
188
+ catch (error) {
189
+ const message = error instanceof Error ? error.message : String(error);
190
+ logger.warn("[TranscribeWebSocket] stream failed", { error: message });
191
+ send({ type: "error", message, recoverable: false });
192
+ if (client.readyState === WebSocket.OPEN) {
193
+ client.close(1011, "transcription failed");
194
+ }
195
+ }
196
+ };
197
+ client.on("message", (data, isBinary) => {
198
+ const frame = Buffer.isBuffer(data)
199
+ ? data
200
+ : Array.isArray(data)
201
+ ? Buffer.concat(data)
202
+ : Buffer.from(data);
203
+ if (!started) {
204
+ if (isBinary) {
205
+ send({
206
+ type: "error",
207
+ message: "The first frame must be a JSON config (text), before any audio",
208
+ recoverable: false,
209
+ });
210
+ client.close(1008, "config first");
211
+ return;
212
+ }
213
+ let parsed;
214
+ try {
215
+ parsed = JSON.parse(frame.toString("utf-8"));
216
+ }
217
+ catch {
218
+ parsed = undefined;
219
+ }
220
+ const config = TranscribeStreamConfigSchema.safeParse(parsed);
221
+ if (!config.success) {
222
+ send({
223
+ type: "error",
224
+ message: `Invalid config: ${config.error.issues
225
+ .map((issue) => `${issue.path.join(".") || "config"} ${issue.message}`)
226
+ .join("; ")}`,
227
+ recoverable: false,
228
+ });
229
+ client.close(1008, "invalid config");
230
+ return;
231
+ }
232
+ started = true;
233
+ void run(config.data);
234
+ return;
235
+ }
236
+ if (isBinary) {
237
+ queue.push(frame);
238
+ if (queue.queuedBytes > MAX_QUEUED_BYTES) {
239
+ send({
240
+ type: "error",
241
+ message: "Audio is arriving faster than it can be transcribed",
242
+ recoverable: false,
243
+ });
244
+ queue.end();
245
+ client.close(1009, "backlog");
246
+ }
247
+ return;
248
+ }
249
+ // Any text frame after the config ends the audio ({"type":"end"}).
250
+ queue.end();
251
+ });
252
+ client.on("close", () => queue.end());
253
+ client.on("error", () => queue.end());
254
+ });
255
+ return {
256
+ close: () => {
257
+ server.off("upgrade", onUpgrade);
258
+ for (const client of wss.clients) {
259
+ client.terminate();
260
+ }
261
+ wss.close();
262
+ },
263
+ };
264
+ }
@@ -10,6 +10,7 @@ import type { VideoGenerationResult } from "./multimodal.js";
10
10
  import type { PPTGenerationResult } from "./ppt.js";
11
11
  import type { AvatarResult } from "./avatar.js";
12
12
  import type { MusicResult } from "./music.js";
13
+ import type { TranscribeStreamEvent } from "./transcribe.js";
13
14
  import type { OAuthTokens } from "./auth.js";
14
15
  import type { AccountQuota, ProxyPassthroughAccount } from "./proxy.js";
15
16
  import type { ClaudeSubscriptionTier } from "./subscription.js";
@@ -1144,6 +1145,8 @@ export type ServeCommandArgs = {
1144
1145
  framework?: ServerFramework;
1145
1146
  basePath?: string;
1146
1147
  cors?: boolean;
1148
+ /** Browser origins allowed by CORS (`--cors-origin`, repeatable or comma-separated). */
1149
+ corsOrigin?: string[];
1147
1150
  rateLimit?: number;
1148
1151
  swagger?: boolean;
1149
1152
  config?: string;
@@ -1320,6 +1323,47 @@ export type CliDecideArgs = {
1320
1323
  format?: "text" | "json";
1321
1324
  debug?: boolean;
1322
1325
  };
1326
+ /** Arguments for `neurolink transcribe <file|->`. */
1327
+ export type CliTranscribeArgs = {
1328
+ /** Audio file path, or `-` to read the audio from stdin. */
1329
+ file?: string;
1330
+ provider?: string;
1331
+ model?: string;
1332
+ /** ISO / BCP-47 code, or `auto` (the default) to let the engine detect it. */
1333
+ language?: string;
1334
+ diarize?: boolean;
1335
+ wordTimestamps?: boolean;
1336
+ prompt?: string;
1337
+ /** Path to a JSON array of `{ term, heardAs, meaning }`. */
1338
+ dictionary?: string;
1339
+ /** Inline dictionary entries: `"Term|heard,as|meaning"`. */
1340
+ term?: string[];
1341
+ correct?: boolean;
1342
+ rewriteProvider?: string;
1343
+ rewriteModel?: string;
1344
+ /** `--no-rewrite` sets this to `false`. */
1345
+ rewrite?: boolean;
1346
+ transliterate?: "latin" | "native";
1347
+ secondOpinion?: string;
1348
+ fallback?: string;
1349
+ context?: string;
1350
+ stream?: boolean;
1351
+ format?: "text" | "json" | "srt" | "vtt";
1352
+ timeout?: number;
1353
+ quiet?: boolean;
1354
+ debug?: boolean;
1355
+ };
1356
+ /** One subtitle cue built from segment or word timings (`--format srt|vtt`). */
1357
+ export type CliSubtitleCue = {
1358
+ start: number;
1359
+ end: number;
1360
+ text: string;
1361
+ };
1362
+ /** Prints `transcribeStream()` events for one `--stream` run. */
1363
+ export type CliTranscribeStreamSink = {
1364
+ event(event: TranscribeStreamEvent): void;
1365
+ finish(): void;
1366
+ };
1323
1367
  /** Base options shared across all `neurolink evaluate` subcommands. */
1324
1368
  export type BaseEvaluateArgs = {
1325
1369
  json?: boolean;
@@ -1422,6 +1466,8 @@ export type ServerInstance = {
1422
1466
  registerRouteGroup: (group: import("./server.js").RouteGroup) => void;
1423
1467
  registerMiddleware: (middleware: import("./server.js").MiddlewareDefinition) => void;
1424
1468
  listRoutes?: () => import("./server.js").RouteDefinition[];
1469
+ /** The listening Node http.Server, when the adapter has one to attach WebSocket upgrades to. */
1470
+ getNativeServer?: () => unknown;
1425
1471
  };
1426
1472
  /** Persisted state for a running `neurolink serve` process. */
1427
1473
  export type ServeState = {
@@ -77,6 +77,7 @@ export * from "./tools.js";
77
77
  export * from "./vectorStoreChroma.js";
78
78
  export * from "./vectorStorePinecone.js";
79
79
  export * from "./voice.js";
80
+ export * from "./transcribe.js";
80
81
  export * from "./utilities.js";
81
82
  export * from "./workflow.js";
82
83
  export * from "./ioProcessor.js";