assemblyai 4.33.3 → 4.34.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +22 -0
  2. package/dist/assemblyai.streaming.umd.js +1291 -3
  3. package/dist/assemblyai.streaming.umd.min.js +1 -1
  4. package/dist/assemblyai.umd.js +802 -7
  5. package/dist/assemblyai.umd.min.js +1 -1
  6. package/dist/browser.mjs +775 -5
  7. package/dist/bun.mjs +775 -5
  8. package/dist/deno.mjs +775 -5
  9. package/dist/exports/streaming.d.ts +7 -0
  10. package/dist/index.cjs +802 -7
  11. package/dist/index.mjs +794 -8
  12. package/dist/node.cjs +783 -4
  13. package/dist/node.mjs +775 -5
  14. package/dist/services/index.d.ts +2 -2
  15. package/dist/services/streaming/browser/dual-channel-capture.d.ts +66 -0
  16. package/dist/services/streaming/browser/worklets/pcm16-encoder.d.ts +19 -0
  17. package/dist/services/streaming/energy-vad.d.ts +35 -0
  18. package/dist/services/streaming/index.d.ts +4 -0
  19. package/dist/services/streaming/label-mapper.d.ts +44 -0
  20. package/dist/services/streaming/resampler.d.ts +22 -0
  21. package/dist/services/streaming/service.d.ts +71 -2
  22. package/dist/streaming.browser.mjs +1247 -4
  23. package/dist/streaming.cjs +1287 -3
  24. package/dist/streaming.mjs +1276 -4
  25. package/dist/types/streaming/dual-channel.d.ts +48 -0
  26. package/dist/types/streaming/index.d.ts +140 -4
  27. package/dist/workerd.mjs +775 -5
  28. package/package.json +1 -1
  29. package/src/exports/streaming.ts +7 -0
  30. package/src/services/index.ts +20 -1
  31. package/src/services/streaming/browser/dual-channel-capture.ts +177 -0
  32. package/src/services/streaming/browser/worklets/pcm16-encoder.ts +70 -0
  33. package/src/services/streaming/energy-vad.ts +75 -0
  34. package/src/services/streaming/index.ts +4 -0
  35. package/src/services/streaming/label-mapper.ts +128 -0
  36. package/src/services/streaming/resampler.ts +69 -0
  37. package/src/services/streaming/service.ts +405 -3
  38. package/src/types/streaming/dual-channel.ts +57 -0
  39. package/src/types/streaming/index.ts +144 -1
@@ -1,3 +1,16 @@
1
+ /**
2
+ * Thrown when `DualChannelCapture` is constructed in a non-browser environment
3
+ * (no `globalThis.AudioContext`). The helper is intentionally surfaced from the
4
+ * main entrypoint so the import path is uniform across runtimes; the runtime
5
+ * guard moves to construction time.
6
+ */
7
+ class BrowserOnlyError extends Error {
8
+ constructor(message = "DualChannelCapture requires a browser environment (AudioContext is undefined).") {
9
+ super(message);
10
+ this.name = "BrowserOnlyError";
11
+ }
12
+ }
13
+
1
14
  const { WritableStream } = typeof window !== "undefined"
2
15
  ? window
3
16
  : typeof global !== "undefined"
@@ -67,9 +80,58 @@ const RealtimeErrorMessages = {
67
80
  class RealtimeError extends Error {
68
81
  }
69
82
 
83
+ const StreamingErrorType = {
84
+ BadSampleRate: 4000,
85
+ AuthFailed: 4001,
86
+ InsufficientFunds: 4002,
87
+ FreeTierUser: 4003,
88
+ NonexistentSessionId: 4004,
89
+ SessionExpired: 4008,
90
+ ClosedSession: 4010,
91
+ RateLimited: 4029,
92
+ UniqueSessionViolation: 4030,
93
+ SessionTimeout: 4031,
94
+ AudioTooShort: 4032,
95
+ AudioTooLong: 4033,
96
+ AudioTooSmallToTranscode: 4034,
97
+ BadSchema: 4101,
98
+ TooManyStreams: 4102,
99
+ Reconnected: 4103,
100
+ ServerError: 3005,
101
+ InputValidationError: 3006,
102
+ AudioChunkDurationViolation: 3007,
103
+ MaxSessionDurationExceeded: 3008,
104
+ ConcurrencyLimitExceeded: 3009,
105
+ };
106
+ const StreamingErrorMessages = {
107
+ [StreamingErrorType.ServerError]: "Server error",
108
+ [StreamingErrorType.InputValidationError]: "Input validation error",
109
+ [StreamingErrorType.AudioChunkDurationViolation]: "Audio chunk duration violation",
110
+ [StreamingErrorType.MaxSessionDurationExceeded]: "Session expired: maximum session duration exceeded",
111
+ [StreamingErrorType.ConcurrencyLimitExceeded]: "Too many concurrent sessions",
112
+ [StreamingErrorType.BadSampleRate]: "Sample rate must be a positive integer",
113
+ [StreamingErrorType.AuthFailed]: "Not Authorized",
114
+ [StreamingErrorType.InsufficientFunds]: "Insufficient funds",
115
+ [StreamingErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
116
+ [StreamingErrorType.NonexistentSessionId]: "Session ID does not exist",
117
+ [StreamingErrorType.SessionExpired]: "Session has expired",
118
+ [StreamingErrorType.ClosedSession]: "Session is closed",
119
+ [StreamingErrorType.RateLimited]: "Rate limited",
120
+ [StreamingErrorType.UniqueSessionViolation]: "Unique session violation",
121
+ [StreamingErrorType.SessionTimeout]: "Session Timeout",
122
+ [StreamingErrorType.AudioTooShort]: "Audio too short",
123
+ [StreamingErrorType.AudioTooLong]: "Audio too long",
124
+ [StreamingErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
125
+ [StreamingErrorType.BadSchema]: "Bad schema",
126
+ [StreamingErrorType.TooManyStreams]: "Too many streams",
127
+ [StreamingErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
128
+ };
129
+ class StreamingError extends Error {
130
+ }
131
+
70
132
  const defaultRealtimeUrl = "wss://api.assemblyai.com/v2/realtime/ws";
71
133
  const forceEndOfUtteranceMessage = `{"force_end_utterance":true}`;
72
- const terminateSessionMessage = `{"terminate_session":true}`;
134
+ const terminateSessionMessage$1 = `{"terminate_session":true}`;
73
135
  /**
74
136
  * RealtimeTranscriber connects to the Streaming Speech-to-Text API and lets you transcribe audio in real-time.
75
137
  */
@@ -262,11 +324,11 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c
262
324
  const sessionTerminatedPromise = new Promise((resolve) => {
263
325
  this.sessionTerminatedResolve = resolve;
264
326
  });
265
- this.socket.send(terminateSessionMessage);
327
+ this.socket.send(terminateSessionMessage$1);
266
328
  await sessionTerminatedPromise;
267
329
  }
268
330
  else {
269
- this.socket.send(terminateSessionMessage);
331
+ this.socket.send(terminateSessionMessage$1);
270
332
  }
271
333
  }
272
334
  if (this.socket?.removeAllListeners)
@@ -283,4 +345,1185 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c
283
345
  class RealtimeService extends RealtimeTranscriber {
284
346
  }
285
347
 
286
- export { RealtimeService, RealtimeTranscriber };
348
+ /**
349
+ * Energy-based VAD with adaptive noise-floor tracking and hangover. Pure JS,
350
+ * no dependencies. Suitable for the "which physical channel is speaking" task
351
+ * because the channels are already physically separated at capture — the harder
352
+ * problem (speech vs. non-speech in the wild) is one a customer can swap in a
353
+ * DNN VAD for via the `createVad` parameter.
354
+ *
355
+ * Tuning notes:
356
+ * - thresholdRatio below 2 will treat anything above noise as speech (too sensitive).
357
+ * - thresholdRatio above 6 will miss quiet utterance onsets/offsets.
358
+ * - noiseFloorAlpha above 0.1 makes the floor track quickly (good for non-stationary
359
+ * background) but risks slowly adapting *up* to a sustained low voice.
360
+ */
361
+ class EnergyVad {
362
+ constructor(params = {}) {
363
+ this.hangoverRemaining = 0;
364
+ this.thresholdRatio = params.thresholdRatio ?? 3.0;
365
+ this.noiseFloorAlpha = params.noiseFloorAlpha ?? 0.05;
366
+ this.hangoverFrames = params.hangoverFrames ?? 10;
367
+ this.initialNoiseFloor = params.initialNoiseFloor ?? 1e-4;
368
+ this.noiseFloor = this.initialNoiseFloor;
369
+ }
370
+ process(frame) {
371
+ let sumSq = 0;
372
+ for (let i = 0; i < frame.length; i++) {
373
+ sumSq += frame[i] * frame[i];
374
+ }
375
+ const rms = frame.length > 0 ? Math.sqrt(sumSq / frame.length) : 0;
376
+ const threshold = this.noiseFloor * this.thresholdRatio;
377
+ let active = rms > threshold;
378
+ if (active) {
379
+ this.hangoverRemaining = this.hangoverFrames;
380
+ }
381
+ else if (this.hangoverRemaining > 0) {
382
+ this.hangoverRemaining--;
383
+ active = true;
384
+ // While in hangover, do not update noise floor — RMS may still reflect tail energy.
385
+ }
386
+ else {
387
+ this.noiseFloor =
388
+ this.noiseFloor * (1 - this.noiseFloorAlpha) +
389
+ rms * this.noiseFloorAlpha;
390
+ }
391
+ return { active, energy: rms };
392
+ }
393
+ reset() {
394
+ this.noiseFloor = this.initialNoiseFloor;
395
+ this.hangoverRemaining = 0;
396
+ }
397
+ }
398
+
399
+ /**
400
+ * Append-only ring buffer of VAD frames in stream-relative ms order.
401
+ * `pushFrame` is O(1) amortized; `framesInWindow` is O(n) over kept frames,
402
+ * which is fine for the per-word lookups we do (a 30 s window at 50 frames/s
403
+ * per channel × 2 channels = 3000 entries, scanned once per word).
404
+ *
405
+ * Runtime-agnostic — no DOM or Web Audio dependencies.
406
+ */
407
+ class VadTimeline {
408
+ constructor(windowMs) {
409
+ this.windowMs = windowMs;
410
+ this.frames = [];
411
+ this.head = 0;
412
+ }
413
+ pushFrame(frame) {
414
+ this.frames.push(frame);
415
+ const cutoff = frame.ts - this.windowMs;
416
+ while (this.head < this.frames.length &&
417
+ this.frames[this.head].ts < cutoff) {
418
+ this.head++;
419
+ }
420
+ if (this.head > 1024 && this.head * 2 > this.frames.length) {
421
+ this.frames = this.frames.slice(this.head);
422
+ this.head = 0;
423
+ }
424
+ }
425
+ framesInWindow(startMs, endMs) {
426
+ const out = [];
427
+ for (let i = this.head; i < this.frames.length; i++) {
428
+ const f = this.frames[i];
429
+ if (f.ts < startMs)
430
+ continue;
431
+ if (f.ts > endMs)
432
+ break;
433
+ out.push(f);
434
+ }
435
+ return out;
436
+ }
437
+ clear() {
438
+ this.frames = [];
439
+ this.head = 0;
440
+ }
441
+ }
442
+ /**
443
+ * Sum per-channel active RMS over a window. Returns a Map from channel name
444
+ * to total score. Channels with zero score are omitted.
445
+ */
446
+ function scoreChannels(frames) {
447
+ const scores = new Map();
448
+ for (const f of frames) {
449
+ if (!f.active)
450
+ continue;
451
+ scores.set(f.channel, (scores.get(f.channel) ?? 0) + f.rms);
452
+ }
453
+ return scores;
454
+ }
455
+ /**
456
+ * Decide which channel was dominant during a word's `[start, end]` window.
457
+ *
458
+ * - If no channel has any active VAD energy → `"unknown"`.
459
+ * - If the top channel beats the runner-up by at least `dominanceRatio` → top channel.
460
+ * - Else: top channel wins on absolute score; exact ties → `"unknown"`.
461
+ */
462
+ function attributeWord(word, timeline, params) {
463
+ const scores = scoreChannels(timeline.framesInWindow(word.start, word.end));
464
+ if (scores.size === 0)
465
+ return "unknown";
466
+ const sorted = [...scores.entries()].sort((a, b) => b[1] - a[1]);
467
+ if (sorted.length === 1)
468
+ return sorted[0][0];
469
+ const [topName, topScore] = sorted[0];
470
+ const [runnerName, runnerScore] = sorted[1];
471
+ if (topScore >= params.dominanceRatio * runnerScore)
472
+ return topName;
473
+ if (topScore > runnerScore)
474
+ return topName;
475
+ if (runnerScore > topScore)
476
+ return runnerName;
477
+ return "unknown";
478
+ }
479
+ /**
480
+ * Duration-weighted majority of word channels. `"unknown"` if there are no
481
+ * words, every word resolved to `"unknown"`, or two channels tie exactly.
482
+ */
483
+ function rollUpTurnChannel(words) {
484
+ const totals = new Map();
485
+ for (const w of words) {
486
+ if (!w.channel || w.channel === "unknown")
487
+ continue;
488
+ const dur = Math.max(0, w.end - w.start);
489
+ totals.set(w.channel, (totals.get(w.channel) ?? 0) + dur);
490
+ }
491
+ if (totals.size === 0)
492
+ return "unknown";
493
+ const sorted = [...totals.entries()].sort((a, b) => b[1] - a[1]);
494
+ if (sorted.length === 1)
495
+ return sorted[0][0];
496
+ const [topName, topMs] = sorted[0];
497
+ const [, runnerMs] = sorted[1];
498
+ if (topMs === runnerMs)
499
+ return "unknown";
500
+ return topName;
501
+ }
502
+ /**
503
+ * Mutate `turn` in place: write `turn.words[i].channel` for every word and set
504
+ * `turn.channel` to the duration-weighted rollup.
505
+ *
506
+ * Returns `void` because the transcriber owns the `TurnEvent` ref and forwards
507
+ * the same object to the customer listener — no need to allocate a copy.
508
+ */
509
+ function attributeTurn(turn, timeline, params) {
510
+ for (const w of turn.words) {
511
+ w.channel = attributeWord(w, timeline, params);
512
+ }
513
+ turn.channel = rollUpTurnChannel(turn.words);
514
+ }
515
+
516
+ /**
517
+ * View any `AudioData` (ArrayBuffer / ArrayBufferView / typed array) as a
518
+ * little-endian Int16 sample sequence without copying. Callers must guarantee
519
+ * the underlying byte length is even.
520
+ */
521
+ function toInt16View(audio) {
522
+ // AudioData is ArrayBufferLike per the public type, but in practice callers
523
+ // pass ArrayBuffer or a typed-array view. Handle both without copying.
524
+ if (audio instanceof Int16Array)
525
+ return audio;
526
+ if (ArrayBuffer.isView(audio)) {
527
+ const view = audio;
528
+ return new Int16Array(view.buffer, view.byteOffset, Math.floor(view.byteLength / 2));
529
+ }
530
+ return new Int16Array(audio);
531
+ }
532
+ const defaultStreamingUrl = "wss://streaming.assemblyai.com/v3/ws";
533
+ const terminateSessionMessage = `{"type":"Terminate"}`;
534
+ /**
535
+ * Per-send chunk cap in milliseconds for the dual-channel mixer. The streaming
536
+ * server rejects audio messages longer than 1000 ms (`Input Duration Error`).
537
+ * If a backlog accumulates (e.g. when a browser tab is backgrounded and
538
+ * `setInterval` is throttled to ~1 Hz), `flushMix` loops and emits multiple
539
+ * sends each ≤ this cap until the buffers drain.
540
+ */
541
+ const MAX_CHUNK_MS = 200;
542
+ /**
543
+ * Per-send minimum chunk size in milliseconds. The streaming server also
544
+ * rejects audio messages shorter than 50 ms with the same
545
+ * `Input Duration Error`, so the mixer waits until both per-channel buffers
546
+ * have at least this much accumulated before emitting. Final-flush (close
547
+ * path) bypasses this floor so the trailing partial buffer still gets sent.
548
+ */
549
+ const MIN_CHUNK_MS = 50;
550
+ class StreamingTranscriber {
551
+ constructor(params) {
552
+ this.listeners = {};
553
+ // Dual-channel mode state (allocated only when params.channels is set).
554
+ this.isDualChannel = false;
555
+ this.vadFrameSamples = 0;
556
+ this.minChunkSamples = 0;
557
+ this.maxChunkSamples = 0;
558
+ this.params = {
559
+ ...params,
560
+ websocketBaseUrl: params.websocketBaseUrl || defaultStreamingUrl,
561
+ };
562
+ if ("token" in params && params.token)
563
+ this.token = params.token;
564
+ if ("apiKey" in params && params.apiKey)
565
+ this.apiKey = params.apiKey;
566
+ if (!(this.token || this.apiKey)) {
567
+ throw new Error("API key or temporary token is required.");
568
+ }
569
+ if (params.channels) {
570
+ if (params.channels.length !== 2) {
571
+ throw new Error("StreamingTranscriber.channels must have exactly 2 entries.");
572
+ }
573
+ const names = params.channels.map((c) => c.name);
574
+ if (new Set(names).size !== names.length) {
575
+ throw new Error("StreamingTranscriber.channels names must be unique.");
576
+ }
577
+ this.isDualChannel = true;
578
+ this.channelNames = names;
579
+ const att = params.channelAttribution ?? {};
580
+ this.attributionParams = {
581
+ dominanceRatio: att.dominanceRatio ?? 4,
582
+ timelineWindowMs: att.timelineWindowMs ?? 30_000,
583
+ createVad: att.createVad ?? (() => new EnergyVad()),
584
+ flushIntervalMs: att.flushIntervalMs ?? 50,
585
+ resolveUnknownChannelsMethod: att.resolveUnknownChannelsMethod ?? "window",
586
+ resolutionWindowWords: att.resolutionWindowWords ?? 2,
587
+ speakerHistoryMinRmsEvidence: att.speakerHistoryMinRmsEvidence ?? 0.5,
588
+ speakerHistoryDominanceRatio: att.speakerHistoryDominanceRatio ?? 3,
589
+ };
590
+ if (this.attributionParams.resolveUnknownChannelsMethod ===
591
+ "speaker-history") {
592
+ this.speakerHistory = new Map();
593
+ }
594
+ // 20 ms VAD frames at the transcriber's target sample rate.
595
+ this.vadFrameSamples = Math.max(1, Math.round(params.sampleRate * 0.02));
596
+ this.minChunkSamples = Math.max(1, Math.round(params.sampleRate * (MIN_CHUNK_MS / 1000)));
597
+ this.maxChunkSamples = Math.max(this.minChunkSamples, Math.round(params.sampleRate * (MAX_CHUNK_MS / 1000)));
598
+ this.channelBuffers = new Map(names.map((n) => [n, []]));
599
+ this.channelSamplesReceived = new Map(names.map((n) => [n, 0]));
600
+ this.channelVadFloatBuffers = new Map(names.map((n) => [n, new Float32Array(this.vadFrameSamples)]));
601
+ this.channelVadBufferIdx = new Map(names.map((n) => [n, 0]));
602
+ this.channelVads = new Map(names.map((n) => [n, this.attributionParams.createVad(n)]));
603
+ this.timeline = new VadTimeline(this.attributionParams.timelineWindowMs);
604
+ }
605
+ }
606
+ connectionUrl() {
607
+ const url = new URL(this.params.websocketBaseUrl ?? "");
608
+ if (url.protocol !== "wss:") {
609
+ throw new Error("Invalid protocol, must be wss");
610
+ }
611
+ const searchParams = new URLSearchParams();
612
+ if (this.token) {
613
+ searchParams.set("token", this.token);
614
+ }
615
+ searchParams.set("sample_rate", this.params.sampleRate.toString());
616
+ if (this.params.endOfTurnConfidenceThreshold) {
617
+ searchParams.set("end_of_turn_confidence_threshold", this.params.endOfTurnConfidenceThreshold.toString());
618
+ }
619
+ if (this.params.minEndOfTurnSilenceWhenConfident !== undefined) {
620
+ if (this.params.minTurnSilence !== undefined) {
621
+ console.warn("[Deprecation Warning] Both `minEndOfTurnSilenceWhenConfident` and `minTurnSilence` are set. Using `minTurnSilence`; `minEndOfTurnSilenceWhenConfident` is deprecated.");
622
+ }
623
+ else {
624
+ console.warn("[Deprecation Warning] `minEndOfTurnSilenceWhenConfident` is deprecated and will be removed in a future release. Please use `minTurnSilence` instead.");
625
+ }
626
+ }
627
+ const effectiveMinTurnSilence = this.params.minTurnSilence ??
628
+ this.params.minEndOfTurnSilenceWhenConfident;
629
+ if (effectiveMinTurnSilence !== undefined) {
630
+ searchParams.set("min_turn_silence", effectiveMinTurnSilence.toString());
631
+ }
632
+ if (this.params.maxTurnSilence) {
633
+ searchParams.set("max_turn_silence", this.params.maxTurnSilence.toString());
634
+ }
635
+ if (this.params.vadThreshold !== undefined) {
636
+ searchParams.set("vad_threshold", this.params.vadThreshold.toString());
637
+ }
638
+ if (this.params.formatTurns) {
639
+ searchParams.set("format_turns", this.params.formatTurns.toString());
640
+ }
641
+ if (this.params.encoding) {
642
+ searchParams.set("encoding", this.params.encoding.toString());
643
+ }
644
+ if (this.params.keytermsPrompt) {
645
+ searchParams.set("keyterms_prompt", JSON.stringify(this.params.keytermsPrompt));
646
+ }
647
+ else if (this.params.keyterms) {
648
+ console.warn("[Deprecation Warning] `keyterms` is deprecated and will be removed in a future release. Please use `keytermsPrompt` instead.");
649
+ searchParams.set("keyterms_prompt", JSON.stringify(this.params.keyterms));
650
+ }
651
+ if (this.params.prompt) {
652
+ searchParams.set("prompt", this.params.prompt);
653
+ }
654
+ if (this.params.agentContext) {
655
+ searchParams.set("agent_context", this.params.agentContext);
656
+ }
657
+ if (this.params.filterProfanity) {
658
+ searchParams.set("filter_profanity", this.params.filterProfanity.toString());
659
+ }
660
+ if (this.params.speechModel === "u3-pro") {
661
+ console.warn("[Deprecation Warning] The speech model `u3-pro` is deprecated and will be removed in a future release. Please use `u3-rt-pro` instead.");
662
+ }
663
+ if (this.params.speechModel !== undefined) {
664
+ searchParams.set("speech_model", this.params.speechModel.toString());
665
+ }
666
+ if (this.params.languageDetection !== undefined) {
667
+ searchParams.set("language_detection", this.params.languageDetection.toString());
668
+ }
669
+ if (this.params.domain) {
670
+ searchParams.set("domain", this.params.domain);
671
+ }
672
+ if (this.params.inactivityTimeout !== undefined) {
673
+ searchParams.set("inactivity_timeout", this.params.inactivityTimeout.toString());
674
+ }
675
+ if (this.params.speakerLabels !== undefined) {
676
+ searchParams.set("speaker_labels", this.params.speakerLabels.toString());
677
+ }
678
+ if (this.params.maxSpeakers !== undefined) {
679
+ searchParams.set("max_speakers", this.params.maxSpeakers.toString());
680
+ }
681
+ if (this.params.voiceFocus) {
682
+ searchParams.set("voice_focus", this.params.voiceFocus);
683
+ }
684
+ if (this.params.voiceFocusThreshold !== undefined) {
685
+ searchParams.set("voice_focus_threshold", this.params.voiceFocusThreshold.toString());
686
+ }
687
+ if (this.params.continuousPartials !== undefined) {
688
+ searchParams.set("continuous_partials", this.params.continuousPartials.toString());
689
+ }
690
+ if (this.params.interruptionDelay !== undefined) {
691
+ searchParams.set("interruption_delay", this.params.interruptionDelay.toString());
692
+ }
693
+ if (this.params.turnLeftPadMs !== undefined) {
694
+ searchParams.set("turn_left_pad_ms", this.params.turnLeftPadMs.toString());
695
+ }
696
+ if (this.params.customerSupportAudioCapture) {
697
+ console.warn("`customerSupportAudioCapture=true` will record session audio. Only enable this when explicitly coordinating with AssemblyAI support.");
698
+ // The server's canonical wire name is `_customer_support_audio_capture`
699
+ // (leading underscore = "not officially supported / unstable"). The
700
+ // server also accepts `customer_support_audio_capture` via
701
+ // `populate_by_name=True`, but we send the underscore form to honor
702
+ // the server's stability marker.
703
+ searchParams.set("_customer_support_audio_capture", this.params.customerSupportAudioCapture.toString());
704
+ }
705
+ if (this.params.webhookUrl) {
706
+ searchParams.set("webhook_url", this.params.webhookUrl);
707
+ }
708
+ if (this.params.webhookAuthHeaderName) {
709
+ searchParams.set("webhook_auth_header_name", this.params.webhookAuthHeaderName);
710
+ }
711
+ if (this.params.webhookAuthHeaderValue) {
712
+ searchParams.set("webhook_auth_header_value", this.params.webhookAuthHeaderValue);
713
+ }
714
+ if (this.params.includePartialTurns !== undefined) {
715
+ searchParams.set("include_partial_turns", this.params.includePartialTurns.toString());
716
+ }
717
+ if (this.params.redactPii !== undefined) {
718
+ searchParams.set("redact_pii", this.params.redactPii.toString());
719
+ }
720
+ if (this.params.redactPiiPolicies !== undefined) {
721
+ searchParams.set("redact_pii_policies", JSON.stringify(this.params.redactPiiPolicies));
722
+ }
723
+ if (this.params.redactPiiSub !== undefined) {
724
+ searchParams.set("redact_pii_sub", this.params.redactPiiSub);
725
+ }
726
+ if (this.params.mode !== undefined) {
727
+ searchParams.set("mode", this.params.mode);
728
+ }
729
+ if (this.params.llmGateway !== undefined) {
730
+ searchParams.set("llm_gateway", JSON.stringify(this.params.llmGateway));
731
+ }
732
+ url.search = searchParams.toString();
733
+ return url;
734
+ }
735
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
736
+ on(event, listener) {
737
+ this.listeners[event] = listener;
738
+ }
739
+ connect() {
740
+ return new Promise((resolve) => {
741
+ if (this.socket) {
742
+ throw new Error("Already connected");
743
+ }
744
+ const url = this.connectionUrl();
745
+ if (this.token) {
746
+ this.socket = factory(url.toString());
747
+ }
748
+ else {
749
+ {
750
+ console.warn(`API key authentication is not supported for the StreamingTranscriber in browser environment. Use temporary token authentication instead.
751
+ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/compat.md#browser-compatibility.`);
752
+ }
753
+ this.socket = factory(url.toString(), {
754
+ headers: { Authorization: this.apiKey },
755
+ });
756
+ }
757
+ this.socket.binaryType = "arraybuffer";
758
+ this.socket.onopen = () => { };
759
+ this.socket.onclose = ({ code, reason }) => {
760
+ if (!reason) {
761
+ if (code in StreamingErrorMessages) {
762
+ reason = StreamingErrorMessages[code];
763
+ }
764
+ }
765
+ // Stop the flush timer when the socket is gone (server-initiated close,
766
+ // network drop, etc.) — otherwise subsequent ticks call send() on a
767
+ // closed socket and spam the error listener.
768
+ if (this.flushTimer) {
769
+ clearInterval(this.flushTimer);
770
+ this.flushTimer = undefined;
771
+ }
772
+ this.listeners.close?.(code, reason);
773
+ };
774
+ this.socket.onerror = (event) => {
775
+ if (event.error)
776
+ this.listeners.error?.(event.error);
777
+ else
778
+ this.listeners.error?.(new Error(event.message));
779
+ };
780
+ this.socket.onmessage = ({ data }) => {
781
+ const message = JSON.parse(data.toString());
782
+ if ("error" in message) {
783
+ const err = new StreamingError(message.error);
784
+ if ("error_code" in message) {
785
+ err.code =
786
+ message.error_code;
787
+ }
788
+ this.listeners.error?.(err);
789
+ return;
790
+ }
791
+ switch (message.type) {
792
+ case "Begin": {
793
+ resolve(message);
794
+ this.listeners.open?.(message);
795
+ break;
796
+ }
797
+ case "Turn": {
798
+ if (this.isDualChannel && this.timeline && this.attributionParams) {
799
+ attributeTurn(message, this.timeline, {
800
+ dominanceRatio: this.attributionParams.dominanceRatio,
801
+ });
802
+ switch (this.attributionParams.resolveUnknownChannelsMethod) {
803
+ case "window":
804
+ this.resolveUnknownChannelsByWindow(message);
805
+ break;
806
+ case "speaker-history":
807
+ this.resolveUnknownChannelsBySpeakerHistory(message);
808
+ break;
809
+ }
810
+ }
811
+ this.listeners.turn?.(message);
812
+ break;
813
+ }
814
+ case "SpeechStarted": {
815
+ this.listeners.speechStarted?.(message);
816
+ break;
817
+ }
818
+ case "LLMGatewayResponse": {
819
+ this.listeners.llmGatewayResponse?.(message);
820
+ break;
821
+ }
822
+ case "SpeakerRevision": {
823
+ this.listeners.speakerRevision?.(message);
824
+ break;
825
+ }
826
+ case "Warning": {
827
+ const warning = message;
828
+ console.warn(`Streaming warning (code=${warning.warning_code}): ${warning.warning}`);
829
+ this.listeners.warning?.(warning);
830
+ break;
831
+ }
832
+ case "Termination": {
833
+ this.sessionTerminatedResolve?.();
834
+ break;
835
+ }
836
+ }
837
+ };
838
+ });
839
+ }
840
+ /**
841
+ * Returns a WritableStream that pumps PCM chunks into `sendAudio`. Single-channel
842
+ * only — in dual-channel mode use `sendAudio(pcm, { channel })` directly, since
843
+ * `WritableStream` has no place to carry a channel tag.
844
+ */
845
+ stream() {
846
+ return new WritableStream({
847
+ write: (chunk) => {
848
+ this.sendAudio(chunk);
849
+ },
850
+ });
851
+ }
852
+ /**
853
+ * Send PCM audio.
854
+ *
855
+ * In single-channel mode, `audio` is forwarded directly to the WebSocket and
856
+ * `options` is ignored.
857
+ *
858
+ * In dual-channel mode (when `channels` is configured), `options.channel` is
859
+ * REQUIRED and must match one of the declared channel names. Per-channel PCM is
860
+ * fed into that channel's VAD, accumulated into a per-channel ring buffer, and
861
+ * a scheduled flush (`channelAttribution.flushIntervalMs`, default 50ms) mixes
862
+ * the buffers into mono before sending to the WebSocket.
863
+ */
864
+ sendAudio(audio, options) {
865
+ if (!this.isDualChannel) {
866
+ this.send(audio);
867
+ return;
868
+ }
869
+ if (!options?.channel) {
870
+ throw new Error("StreamingTranscriber is in dual-channel mode; sendAudio requires { channel }.");
871
+ }
872
+ if (!this.channelNames.includes(options.channel)) {
873
+ throw new Error(`Unknown channel "${options.channel}"; declared channels: ${this.channelNames.join(", ")}.`);
874
+ }
875
+ this.ingestChannelAudio(options.channel, audio);
876
+ }
877
+ ingestChannelAudio(name, audio) {
878
+ const samples = toInt16View(audio);
879
+ const buf = this.channelBuffers.get(name);
880
+ const vadBuf = this.channelVadFloatBuffers.get(name);
881
+ let vadIdx = this.channelVadBufferIdx.get(name);
882
+ let received = this.channelSamplesReceived.get(name);
883
+ const vad = this.channelVads.get(name);
884
+ const sampleRate = this.params.sampleRate;
885
+ const frameSize = this.vadFrameSamples;
886
+ for (let i = 0; i < samples.length; i++) {
887
+ const s = samples[i];
888
+ buf.push(s);
889
+ vadBuf[vadIdx++] = s / 0x8000;
890
+ received++;
891
+ if (vadIdx === frameSize) {
892
+ const result = vad.process(vadBuf);
893
+ const frame = {
894
+ ts: (received / sampleRate) * 1000,
895
+ channel: name,
896
+ active: result.active,
897
+ rms: result.energy,
898
+ };
899
+ this.timeline.pushFrame(frame);
900
+ this.listeners.vad?.(frame);
901
+ vadIdx = 0;
902
+ }
903
+ }
904
+ this.channelVadBufferIdx.set(name, vadIdx);
905
+ this.channelSamplesReceived.set(name, received);
906
+ if (!this.flushTimer)
907
+ this.startFlushTimer();
908
+ }
909
+ startFlushTimer() {
910
+ this.flushTimer = setInterval(() => this.flushMix(), this.attributionParams.flushIntervalMs);
911
+ }
912
+ flushMix(force = false) {
913
+ if (!this.channelNames || !this.channelBuffers)
914
+ return;
915
+ const bufs = this.channelNames.map((n) => this.channelBuffers.get(n));
916
+ const divisor = bufs.length;
917
+ // Loop so a backlog (e.g. accumulated while a browser tab was throttled in
918
+ // the background) drains as multiple sends, each capped at MAX_CHUNK_MS.
919
+ // Without the cap a single message could exceed the server's 1000 ms input
920
+ // duration limit and be rejected with code 3007.
921
+ for (;;) {
922
+ let mixLen = Infinity;
923
+ for (const b of bufs)
924
+ if (b.length < mixLen)
925
+ mixLen = b.length;
926
+ if (!Number.isFinite(mixLen) || mixLen === 0)
927
+ return;
928
+ // The streaming server rejects audio messages shorter than 50 ms with
929
+ // `Input Duration Error`. Wait until both per-channel buffers have at
930
+ // least minChunkSamples worth queued before emitting. The `force` path
931
+ // (final flush on close) bypasses this so the trailing partial buffer
932
+ // still gets through.
933
+ if (!force && mixLen < this.minChunkSamples)
934
+ return;
935
+ if (mixLen > this.maxChunkSamples)
936
+ mixLen = this.maxChunkSamples;
937
+ const out = new Int16Array(mixLen);
938
+ for (let i = 0; i < mixLen; i++) {
939
+ let sum = 0;
940
+ for (let c = 0; c < divisor; c++)
941
+ sum += bufs[c][i];
942
+ const avg = Math.round(sum / divisor);
943
+ out[i] = avg < -32768 ? -32768 : avg > 32767 ? 32767 : avg;
944
+ }
945
+ for (const b of bufs)
946
+ b.splice(0, mixLen);
947
+ try {
948
+ this.send(out.buffer);
949
+ }
950
+ catch (err) {
951
+ this.listeners.error?.(err);
952
+ return;
953
+ }
954
+ }
955
+ }
956
+ /**
957
+ * Fill in words whose per-word VAD attribution was `"unknown"` by looking
958
+ * at the dominant non-`"unknown"` channel among ±N neighbors in the same
959
+ * turn. Words with no non-`"unknown"` neighbors stay `"unknown"`. Confident
960
+ * per-word VAD decisions are never modified.
961
+ *
962
+ * Local temporal heuristic — ignores `speaker_label`, so it works even when
963
+ * AAI's diarization re-uses the same label for two physically distinct
964
+ * voices. Each resolved word gets `channelResolved: true` so downstream
965
+ * renderers can distinguish inferred channels from directly-measured ones.
966
+ */
967
+ resolveUnknownChannelsByWindow(turn) {
968
+ if (!this.attributionParams)
969
+ return;
970
+ const window = this.attributionParams.resolutionWindowWords;
971
+ const words = turn.words;
972
+ let mutated = false;
973
+ for (let i = 0; i < words.length; i++) {
974
+ if (words[i].channel !== "unknown")
975
+ continue;
976
+ const tally = new Map();
977
+ const lo = Math.max(0, i - window);
978
+ const hi = Math.min(words.length - 1, i + window);
979
+ for (let j = lo; j <= hi; j++) {
980
+ if (j === i)
981
+ continue;
982
+ const ch = words[j].channel;
983
+ if (!ch || ch === "unknown")
984
+ continue;
985
+ tally.set(ch, (tally.get(ch) ?? 0) + 1);
986
+ }
987
+ if (tally.size === 0)
988
+ continue;
989
+ // Pick the dominant neighbor channel. Ties → leave `"unknown"` (rare;
990
+ // would require an equal count of mic and system neighbors).
991
+ let top;
992
+ let topCount = 0;
993
+ let tied = false;
994
+ for (const [name, count] of tally) {
995
+ if (count > topCount) {
996
+ top = name;
997
+ topCount = count;
998
+ tied = false;
999
+ }
1000
+ else if (count === topCount) {
1001
+ tied = true;
1002
+ }
1003
+ }
1004
+ if (top && !tied) {
1005
+ words[i].channel = top;
1006
+ words[i].channelResolved = true;
1007
+ mutated = true;
1008
+ }
1009
+ }
1010
+ // Recompute the rollup only if any per-word channel changed.
1011
+ if (mutated)
1012
+ turn.channel = rollUpTurnChannel(words);
1013
+ }
1014
+ /**
1015
+ * Fill `"unknown"` words by looking up the speaker's session-wide channel
1016
+ * evidence. For each `speaker_label`, sums active VAD frame RMS per channel
1017
+ * across every word the speaker has uttered to date. A speaker is
1018
+ * "resolvable" if their total evidence clears
1019
+ * `speakerHistoryMinRmsEvidence` and their top channel exceeds the
1020
+ * runner-up by `speakerHistoryDominanceRatio`.
1021
+ *
1022
+ * Only touches `"unknown"` words. Confident per-word VAD decisions are
1023
+ * never modified. `speaker_label` is never modified.
1024
+ */
1025
+ resolveUnknownChannelsBySpeakerHistory(turn) {
1026
+ if (!this.timeline || !this.attributionParams || !this.speakerHistory)
1027
+ return;
1028
+ const minEvidence = this.attributionParams.speakerHistoryMinRmsEvidence;
1029
+ const dominanceRatio = this.attributionParams.speakerHistoryDominanceRatio;
1030
+ // 1. Accumulate evidence from this turn's words.
1031
+ for (const w of turn.words) {
1032
+ if (!w.speaker)
1033
+ continue;
1034
+ const frames = this.timeline.framesInWindow(w.start, w.end);
1035
+ let entry = this.speakerHistory.get(w.speaker);
1036
+ if (!entry) {
1037
+ entry = new Map();
1038
+ this.speakerHistory.set(w.speaker, entry);
1039
+ }
1040
+ for (const f of frames) {
1041
+ if (!f.active)
1042
+ continue;
1043
+ entry.set(f.channel, (entry.get(f.channel) ?? 0) + f.rms);
1044
+ }
1045
+ }
1046
+ // 2. Fill unknown words whose speakers have dominant evidence.
1047
+ let mutated = false;
1048
+ for (const w of turn.words) {
1049
+ if (w.channel !== "unknown" || !w.speaker)
1050
+ continue;
1051
+ const entry = this.speakerHistory.get(w.speaker);
1052
+ if (!entry || entry.size === 0)
1053
+ continue;
1054
+ let total = 0;
1055
+ let topName;
1056
+ let topScore = 0;
1057
+ let runnerScore = 0;
1058
+ for (const [name, score] of entry) {
1059
+ total += score;
1060
+ if (score > topScore) {
1061
+ runnerScore = topScore;
1062
+ topScore = score;
1063
+ topName = name;
1064
+ }
1065
+ else if (score > runnerScore) {
1066
+ runnerScore = score;
1067
+ }
1068
+ }
1069
+ if (total < minEvidence)
1070
+ continue;
1071
+ if (runnerScore > 0 && topScore < dominanceRatio * runnerScore)
1072
+ continue;
1073
+ if (topName) {
1074
+ w.channel = topName;
1075
+ w.channelResolved = true;
1076
+ mutated = true;
1077
+ }
1078
+ }
1079
+ if (mutated)
1080
+ turn.channel = rollUpTurnChannel(turn.words);
1081
+ }
1082
+ /**
1083
+ * Update the streaming configuration mid-stream.
1084
+ * @param config - The configuration parameters to update
1085
+ */
1086
+ updateConfiguration(config) {
1087
+ const { min_end_of_turn_silence_when_confident, min_turn_silence, ...rest } = config;
1088
+ if (min_end_of_turn_silence_when_confident !== undefined) {
1089
+ if (min_turn_silence !== undefined) {
1090
+ console.warn("[Deprecation Warning] Both `min_end_of_turn_silence_when_confident` and `min_turn_silence` are set. Using `min_turn_silence`; `min_end_of_turn_silence_when_confident` is deprecated.");
1091
+ }
1092
+ else {
1093
+ console.warn("[Deprecation Warning] `min_end_of_turn_silence_when_confident` is deprecated and will be removed in a future release. Please use `min_turn_silence` instead.");
1094
+ }
1095
+ }
1096
+ const effective = min_turn_silence ?? min_end_of_turn_silence_when_confident;
1097
+ const message = {
1098
+ type: "UpdateConfiguration",
1099
+ ...rest,
1100
+ ...(effective !== undefined ? { min_turn_silence: effective } : {}),
1101
+ };
1102
+ this.send(JSON.stringify(message));
1103
+ }
1104
+ /**
1105
+ * Force the current turn to end immediately.
1106
+ */
1107
+ forceEndpoint() {
1108
+ const message = {
1109
+ type: "ForceEndpoint",
1110
+ };
1111
+ this.send(JSON.stringify(message));
1112
+ }
1113
+ send(data) {
1114
+ if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
1115
+ throw new Error("Socket is not open for communication");
1116
+ }
1117
+ this.socket.send(data);
1118
+ }
1119
+ async close(waitForSessionTermination = true) {
1120
+ if (this.flushTimer) {
1121
+ clearInterval(this.flushTimer);
1122
+ this.flushTimer = undefined;
1123
+ // Best-effort: drain any final partial mix so the server gets the tail.
1124
+ // Bypass the 50ms floor here since this is the last flush; if the tail
1125
+ // is <50ms the server will reject that single message, but we'd lose
1126
+ // the audio either way.
1127
+ this.flushMix(true);
1128
+ }
1129
+ if (this.socket) {
1130
+ if (this.socket.readyState === this.socket.OPEN) {
1131
+ if (waitForSessionTermination) {
1132
+ const sessionTerminatedPromise = new Promise((resolve) => {
1133
+ this.sessionTerminatedResolve = resolve;
1134
+ });
1135
+ this.socket.send(terminateSessionMessage);
1136
+ await sessionTerminatedPromise;
1137
+ }
1138
+ else {
1139
+ this.socket.send(terminateSessionMessage);
1140
+ }
1141
+ }
1142
+ if (this.socket?.removeAllListeners)
1143
+ this.socket.removeAllListeners();
1144
+ this.socket.close();
1145
+ }
1146
+ this.listeners = {};
1147
+ this.socket = undefined;
1148
+ }
1149
+ }
1150
+
1151
+ const DEFAULT_FETCH_INIT = {
1152
+ cache: "no-store",
1153
+ };
1154
+
1155
+ const buildUserAgent = (userAgent) => defaultUserAgentString +
1156
+ (userAgent === false
1157
+ ? ""
1158
+ : " AssemblyAI/1.0 (" +
1159
+ Object.entries({ ...defaultUserAgent, ...userAgent })
1160
+ .map(([key, item]) => item ? `${key}=${item.name}/${item.version}` : "")
1161
+ .join(" ") +
1162
+ ")");
1163
+ let defaultUserAgentString = "";
1164
+ if (typeof navigator !== "undefined" && navigator.userAgent) {
1165
+ defaultUserAgentString += navigator.userAgent;
1166
+ }
1167
+ const defaultUserAgent = {
1168
+ sdk: { name: "JavaScript", version: "4.34.4" },
1169
+ };
1170
+ if (typeof process !== "undefined") {
1171
+ if (process.versions.node && defaultUserAgentString.indexOf("Node") === -1) {
1172
+ defaultUserAgent.runtime_env = {
1173
+ name: "Node",
1174
+ version: process.versions.node,
1175
+ };
1176
+ }
1177
+ if (process.versions.bun && defaultUserAgentString.indexOf("Bun") === -1) {
1178
+ defaultUserAgent.runtime_env = {
1179
+ name: "Bun",
1180
+ version: process.versions.bun,
1181
+ };
1182
+ }
1183
+ }
1184
+ if (typeof Deno !== "undefined") {
1185
+ if (process.versions.bun && defaultUserAgentString.indexOf("Deno") === -1) {
1186
+ defaultUserAgent.runtime_env = { name: "Deno", version: Deno.version.deno };
1187
+ }
1188
+ }
1189
+
1190
+ /**
1191
+ * Base class for services that communicate with the API.
1192
+ */
1193
+ class BaseService {
1194
+ /**
1195
+ * Create a new service.
1196
+ * @param params - The parameters to use for the service.
1197
+ */
1198
+ constructor(params) {
1199
+ this.params = params;
1200
+ if (params.userAgent === false) {
1201
+ this.userAgent = undefined;
1202
+ }
1203
+ else {
1204
+ this.userAgent = buildUserAgent(params.userAgent || {});
1205
+ }
1206
+ }
1207
+ async fetch(input, init) {
1208
+ init = { ...DEFAULT_FETCH_INIT, ...init };
1209
+ let headers = {
1210
+ Authorization: this.params.apiKey,
1211
+ "Content-Type": "application/json",
1212
+ };
1213
+ if (DEFAULT_FETCH_INIT?.headers)
1214
+ headers = { ...headers, ...DEFAULT_FETCH_INIT.headers };
1215
+ if (init?.headers)
1216
+ headers = { ...headers, ...init.headers };
1217
+ if (this.userAgent) {
1218
+ headers["User-Agent"] = this.userAgent;
1219
+ {
1220
+ // chromium browsers have a bug where the user agent can't be modified
1221
+ if (typeof window !== "undefined" && "chrome" in window) {
1222
+ headers["AssemblyAI-Agent"] =
1223
+ this.userAgent;
1224
+ }
1225
+ }
1226
+ }
1227
+ init.headers = headers;
1228
+ if (!input.startsWith("http"))
1229
+ input = this.params.baseUrl + input;
1230
+ const response = await fetch(input, init);
1231
+ if (response.status >= 400) {
1232
+ let json;
1233
+ const text = await response.text();
1234
+ if (text) {
1235
+ try {
1236
+ json = JSON.parse(text);
1237
+ }
1238
+ catch {
1239
+ /* empty */
1240
+ }
1241
+ if (json?.error)
1242
+ throw new Error(json.error);
1243
+ throw new Error(text);
1244
+ }
1245
+ throw new Error(`HTTP Error: ${response.status} ${response.statusText}`);
1246
+ }
1247
+ return response;
1248
+ }
1249
+ async fetchJson(input, init) {
1250
+ const response = await this.fetch(input, init);
1251
+ return response.json();
1252
+ }
1253
+ }
1254
+
1255
+ class StreamingTranscriberFactory extends BaseService {
1256
+ constructor(params) {
1257
+ super(params);
1258
+ this.baseServiceParams = params;
1259
+ }
1260
+ transcriber(params) {
1261
+ const serviceParams = { ...params };
1262
+ if (!serviceParams.token && !serviceParams.apiKey) {
1263
+ serviceParams.apiKey = this.baseServiceParams.apiKey;
1264
+ }
1265
+ return new StreamingTranscriber(serviceParams);
1266
+ }
1267
+ async createTemporaryToken(params) {
1268
+ const searchParams = new URLSearchParams();
1269
+ // Add each param to the search params
1270
+ Object.entries(params).forEach(([key, value]) => {
1271
+ if (value !== undefined && value !== null) {
1272
+ searchParams.append(key, String(value));
1273
+ }
1274
+ });
1275
+ const queryString = searchParams.toString();
1276
+ const url = queryString ? `/v3/token?${queryString}` : "/v3/token";
1277
+ const data = await this.fetchJson(url, {
1278
+ method: "GET",
1279
+ });
1280
+ return data.token;
1281
+ }
1282
+ }
1283
+ class StreamingServiceFactory extends StreamingTranscriberFactory {
1284
+ }
1285
+
1286
+ /**
1287
+ * AudioWorklet processor that ingests mono Float32 audio at the AudioContext's
1288
+ * native sample rate, resamples to `targetRate` (linear interpolation, stateful
1289
+ * across `process()` calls), packs to little-endian Int16 PCM, and posts
1290
+ * fixed-size chunks via `port.postMessage` with a running `samplesSent` counter.
1291
+ *
1292
+ * `samplesSent` is in **target-rate samples**, so the main thread can derive a
1293
+ * stream-relative timestamp = `samplesSent / targetRate * 1000` (ms) — the same
1294
+ * frame AAI uses for `StreamingWord.start` / `.end`.
1295
+ *
1296
+ * Defined as a string so it can be registered via a Blob URL — the SDK ships as
1297
+ * a single ESM file, so a separate `.js` worklet asset isn't viable.
1298
+ */
1299
+ const pcm16EncoderWorkletSource = `
1300
+ class Pcm16EncoderProcessor extends AudioWorkletProcessor {
1301
+ constructor(options) {
1302
+ super();
1303
+ const opts = (options && options.processorOptions) || {};
1304
+ this.targetRate = opts.targetRate || 16000;
1305
+ this.chunkMs = opts.chunkMs || 50;
1306
+ this.ratio = sampleRate / this.targetRate;
1307
+ this.chunkSize = Math.round(this.targetRate * this.chunkMs / 1000);
1308
+ this.buffer = new Int16Array(this.chunkSize);
1309
+ this.bufferIdx = 0;
1310
+ this.samplesSent = 0;
1311
+ this.lastSample = 0;
1312
+ this.fractional = 0;
1313
+ }
1314
+
1315
+ process(inputs) {
1316
+ const input = inputs[0];
1317
+ if (!input || input.length === 0 || !input[0] || input[0].length === 0) {
1318
+ return true;
1319
+ }
1320
+ const mono = input[0];
1321
+ let pos = this.fractional;
1322
+ while (pos < mono.length) {
1323
+ const i = Math.floor(pos);
1324
+ const frac = pos - i;
1325
+ const a = i === 0 ? this.lastSample : mono[i - 1];
1326
+ const b = mono[i];
1327
+ const sample = a + (b - a) * frac;
1328
+ const clamped = sample < -1 ? -1 : sample > 1 ? 1 : sample;
1329
+ this.buffer[this.bufferIdx++] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
1330
+ if (this.bufferIdx === this.chunkSize) {
1331
+ const out = new Int16Array(this.chunkSize);
1332
+ out.set(this.buffer);
1333
+ this.samplesSent += this.chunkSize;
1334
+ this.port.postMessage(
1335
+ { pcm: out.buffer, samplesSent: this.samplesSent },
1336
+ [out.buffer],
1337
+ );
1338
+ this.bufferIdx = 0;
1339
+ }
1340
+ pos += this.ratio;
1341
+ }
1342
+ this.lastSample = mono[mono.length - 1];
1343
+ this.fractional = pos - mono.length;
1344
+ return true;
1345
+ }
1346
+ }
1347
+ registerProcessor("aai-pcm16-encoder", Pcm16EncoderProcessor);
1348
+ `;
1349
+ const PCM16_ENCODER_PROCESSOR_NAME = "aai-pcm16-encoder";
1350
+
1351
+ const DEFAULT_TARGET_RATE = 16_000;
1352
+ const DEFAULT_CHUNK_MS = 50;
1353
+ const MIC_CHANNEL = "mic";
1354
+ const SYSTEM_CHANNEL = "system";
1355
+ /**
1356
+ * Browser-only adapter that pumps two `MediaStream`s into a `StreamingTranscriber`
1357
+ * configured for dual-channel mode. Each `MediaStream` runs through its own
1358
+ * `pcm16-encoder` AudioWorklet (resample to `targetSampleRate`, encode to Int16
1359
+ * PCM); each PCM chunk is forwarded via `transcriber.sendAudio(pcm, { channel })`.
1360
+ *
1361
+ * All dual-channel orchestration (mixing, VAD, per-word attribution) lives inside
1362
+ * `StreamingTranscriber` — this class is a pure I/O adapter. Non-browser runtimes
1363
+ * can replicate its job by pushing tagged PCM into `transcriber.sendAudio` directly.
1364
+ *
1365
+ * Caller responsibilities:
1366
+ * - **Echo cancellation** is set at `getUserMedia` time (`audio: { echoCancellation: true }`).
1367
+ * - **System-audio capture** is platform-dependent. Chrome's `getDisplayMedia({ audio: true })`
1368
+ * captures tab audio (and on Windows, full system audio when sharing the whole screen).
1369
+ * macOS requires a virtual loopback driver (e.g. BlackHole) to expose system audio at all.
1370
+ * - **Token auth.** Construct the transcriber with `token` — API-key auth is unsupported in browsers.
1371
+ * - **Stream ownership.** `stop()` tears down the AudioContext but does NOT stop the
1372
+ * `MediaStreamTrack`s passed in — callers own those.
1373
+ */
1374
+ class DualChannelCapture {
1375
+ constructor(params) {
1376
+ this.running = false;
1377
+ if (typeof globalThis.AudioContext === "undefined") {
1378
+ throw new BrowserOnlyError();
1379
+ }
1380
+ this.params = {
1381
+ micStream: params.micStream,
1382
+ systemStream: params.systemStream,
1383
+ transcriber: params.transcriber,
1384
+ targetSampleRate: params.targetSampleRate ?? DEFAULT_TARGET_RATE,
1385
+ };
1386
+ }
1387
+ on(event, listener) {
1388
+ if (event === "error")
1389
+ this.errorListener = listener;
1390
+ }
1391
+ /**
1392
+ * Wire the capture pipeline and start pumping tagged PCM into the transcriber.
1393
+ * The transcriber must already be connected. Returns once the worklet is
1394
+ * registered and the audio graph is live.
1395
+ */
1396
+ async start() {
1397
+ if (this.running) {
1398
+ throw new Error("DualChannelCapture already started");
1399
+ }
1400
+ this.context = new AudioContext();
1401
+ const blob = new Blob([pcm16EncoderWorkletSource], {
1402
+ type: "application/javascript",
1403
+ });
1404
+ const url = URL.createObjectURL(blob);
1405
+ try {
1406
+ await this.context.audioWorklet.addModule(url);
1407
+ }
1408
+ finally {
1409
+ URL.revokeObjectURL(url);
1410
+ }
1411
+ this.micSource = this.context.createMediaStreamSource(this.params.micStream);
1412
+ this.sysSource = this.context.createMediaStreamSource(this.params.systemStream);
1413
+ this.micEncoder = this.makeEncoder(MIC_CHANNEL);
1414
+ this.sysEncoder = this.makeEncoder(SYSTEM_CHANNEL);
1415
+ this.micSource.connect(this.micEncoder);
1416
+ this.sysSource.connect(this.sysEncoder);
1417
+ this.running = true;
1418
+ }
1419
+ makeEncoder(channel) {
1420
+ const node = new AudioWorkletNode(this.context, PCM16_ENCODER_PROCESSOR_NAME, {
1421
+ numberOfInputs: 1,
1422
+ numberOfOutputs: 0,
1423
+ channelCount: 1,
1424
+ channelCountMode: "explicit",
1425
+ channelInterpretation: "speakers",
1426
+ processorOptions: {
1427
+ targetRate: this.params.targetSampleRate,
1428
+ chunkMs: DEFAULT_CHUNK_MS,
1429
+ },
1430
+ });
1431
+ node.port.onmessage = (e) => {
1432
+ try {
1433
+ this.params.transcriber.sendAudio(e.data.pcm, { channel });
1434
+ }
1435
+ catch (err) {
1436
+ this.errorListener?.(err);
1437
+ }
1438
+ };
1439
+ return node;
1440
+ }
1441
+ /**
1442
+ * Tear down internal nodes and close the AudioContext. Does NOT stop the
1443
+ * caller-provided MediaStream tracks — they remain available for preview UI,
1444
+ * recording, etc. Idempotent.
1445
+ */
1446
+ async stop() {
1447
+ if (!this.running)
1448
+ return;
1449
+ this.running = false;
1450
+ try {
1451
+ this.micEncoder?.port.close();
1452
+ this.sysEncoder?.port.close();
1453
+ this.micEncoder?.disconnect();
1454
+ this.sysEncoder?.disconnect();
1455
+ this.micSource?.disconnect();
1456
+ this.sysSource?.disconnect();
1457
+ }
1458
+ catch {
1459
+ // Disconnecting already-disconnected nodes throws in some browsers; ignore.
1460
+ }
1461
+ if (this.context && this.context.state !== "closed") {
1462
+ await this.context.close();
1463
+ }
1464
+ this.context = undefined;
1465
+ this.micSource = undefined;
1466
+ this.sysSource = undefined;
1467
+ this.micEncoder = undefined;
1468
+ this.sysEncoder = undefined;
1469
+ }
1470
+ }
1471
+
1472
+ /**
1473
+ * Linear-interpolation resampler for streaming Float32 audio. Stateful across
1474
+ * `process()` calls so chunk boundaries don't introduce phase discontinuities:
1475
+ * the last input sample and a fractional read position are carried over.
1476
+ *
1477
+ * Linear interpolation is good enough for ASR ingest — the downstream
1478
+ * StreamingTranscriber band-limits at the target rate anyway, and a polyphase
1479
+ * filter would be overkill in the AudioWorklet hot path. If a customer needs
1480
+ * higher quality they can supply their own VadDetector + bypass the encoder.
1481
+ */
1482
+ class LinearResampler {
1483
+ constructor(sourceRate, targetRate) {
1484
+ this.sourceRate = sourceRate;
1485
+ this.targetRate = targetRate;
1486
+ this.lastSample = 0;
1487
+ this.fractional = 0;
1488
+ if (sourceRate <= 0 || targetRate <= 0) {
1489
+ throw new Error("sourceRate and targetRate must be positive");
1490
+ }
1491
+ this.ratio = sourceRate / targetRate;
1492
+ }
1493
+ process(input) {
1494
+ if (this.sourceRate === this.targetRate) {
1495
+ return input;
1496
+ }
1497
+ // Worst-case output length; we'll slice to actual.
1498
+ const out = new Float32Array(Math.ceil(input.length / this.ratio) + 1);
1499
+ let outIdx = 0;
1500
+ let pos = this.fractional;
1501
+ while (pos < input.length) {
1502
+ const i = Math.floor(pos);
1503
+ const frac = pos - i;
1504
+ const a = i === 0 ? this.lastSample : input[i - 1];
1505
+ const b = input[i];
1506
+ out[outIdx++] = a + (b - a) * frac;
1507
+ pos += this.ratio;
1508
+ }
1509
+ this.lastSample = input[input.length - 1] ?? this.lastSample;
1510
+ this.fractional = pos - input.length;
1511
+ return out.subarray(0, outIdx);
1512
+ }
1513
+ reset() {
1514
+ this.lastSample = 0;
1515
+ this.fractional = 0;
1516
+ }
1517
+ }
1518
+ /** Convert Float32 PCM (-1..1) to little-endian Int16 PCM. */
1519
+ function float32ToPcm16(input) {
1520
+ const out = new ArrayBuffer(input.length * 2);
1521
+ const view = new DataView(out);
1522
+ for (let i = 0; i < input.length; i++) {
1523
+ const clamped = Math.max(-1, Math.min(1, input[i]));
1524
+ view.setInt16(i * 2, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
1525
+ }
1526
+ return out;
1527
+ }
1528
+
1529
+ export { BrowserOnlyError, DualChannelCapture, EnergyVad, LinearResampler, RealtimeService, RealtimeTranscriber, StreamingServiceFactory, StreamingTranscriber, StreamingTranscriberFactory, VadTimeline, attributeTurn, attributeWord, float32ToPcm16, rollUpTurnChannel };