assemblyai 4.33.2 → 4.34.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "assemblyai",
-  "version": "4.33.2",
+  "version": "4.34.0",
   "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.",
   "engines": {
     "node": ">=18"

package/src/exports/streaming.ts

@@ -1,4 +1,11 @@
 export * from "../types/asyncapi.generated";
 export * from "../types/realtime";
 export * from "../types/helpers";
+export * from "../types/streaming/dual-channel";
 export * from "../services/realtime/service";
+export * from "../services/streaming/service";
+export * from "../services/streaming/factory";
+export * from "../services/streaming/browser/dual-channel-capture";
+export * from "../services/streaming/energy-vad";
+export * from "../services/streaming/label-mapper";
+export * from "../services/streaming/resampler";

package/src/services/index.ts

@@ -8,7 +8,18 @@ import {
 } from "./realtime";
 import { TranscriptService } from "./transcripts";
 import { FileService } from "./files";
-import { StreamingTranscriber, StreamingTranscriberFactory } from "./streaming";
+import {
+  StreamingTranscriber,
+  StreamingTranscriberFactory,
+  DualChannelCapture,
+  EnergyVad,
+  LinearResampler,
+  VadTimeline,
+  attributeTurn,
+  attributeWord,
+  rollUpTurnChannel,
+  float32ToPcm16,
+} from "./streaming";
 
 const defaultBaseUrl = "https://api.assemblyai.com";
 const defaultStreamingUrl = "https://streaming.assemblyai.com";
@@ -71,4 +82,12 @@ export {
   TranscriptService,
   FileService,
   StreamingTranscriber,
+  DualChannelCapture,
+  EnergyVad,
+  LinearResampler,
+  VadTimeline,
+  attributeTurn,
+  attributeWord,
+  rollUpTurnChannel,
+  float32ToPcm16,
 };

package/src/services/streaming/browser/dual-channel-capture.ts

@@ -0,0 +1,177 @@
+import { StreamingTranscriber } from "../service";
+import { BrowserOnlyError } from "../../../types/streaming/dual-channel";
+
+export { BrowserOnlyError } from "../../../types/streaming/dual-channel";
+import {
+  PCM16_ENCODER_PROCESSOR_NAME,
+  Pcm16EncoderMessage,
+  pcm16EncoderWorkletSource,
+} from "./worklets/pcm16-encoder";
+
+const DEFAULT_TARGET_RATE = 16_000;
+const DEFAULT_CHUNK_MS = 50;
+const MIC_CHANNEL = "mic";
+const SYSTEM_CHANNEL = "system";
+
+type ErrorListener = (err: Error) => void;
+
+export type DualChannelCaptureParams = {
+  /** Microphone MediaStream. Caller should set `echoCancellation: true` at `getUserMedia` time. */
+  micStream: MediaStream;
+  /** System-audio MediaStream (e.g. `getDisplayMedia({ audio: true })`). */
+  systemStream: MediaStream;
+  /**
+   * The transcriber to push tagged PCM into. MUST be constructed with
+   * `channels: [{ name: "mic" }, { name: "system" }]` so the per-channel
+   * `sendAudio` calls succeed.
+   */
+  transcriber: StreamingTranscriber;
+  /**
+   * Target sample rate sent to the transcriber. Defaults to 16000. The
+   * AudioContext runs at the device's native rate; resampling happens inside
+   * the encoder worklet (forcing `AudioContext({ sampleRate })` is unreliable
+   * across browsers).
+   */
+  targetSampleRate?: number;
+};
+
+/**
+ * Browser-only adapter that pumps two `MediaStream`s into a `StreamingTranscriber`
+ * configured for dual-channel mode. Each `MediaStream` runs through its own
+ * `pcm16-encoder` AudioWorklet (resample to `targetSampleRate`, encode to Int16
+ * PCM); each PCM chunk is forwarded via `transcriber.sendAudio(pcm, { channel })`.
+ *
+ * All dual-channel orchestration (mixing, VAD, per-word attribution) lives inside
+ * `StreamingTranscriber` — this class is a pure I/O adapter. Non-browser runtimes
+ * can replicate its job by pushing tagged PCM into `transcriber.sendAudio` directly.
+ *
+ * Caller responsibilities:
+ * - **Echo cancellation** is set at `getUserMedia` time (`audio: { echoCancellation: true }`).
+ * - **System-audio capture** is platform-dependent. Chrome's `getDisplayMedia({ audio: true })`
+ *   captures tab audio (and on Windows, full system audio when sharing the whole screen).
+ *   macOS requires a virtual loopback driver (e.g. BlackHole) to expose system audio at all.
+ * - **Token auth.** Construct the transcriber with `token` — API-key auth is unsupported in browsers.
+ * - **Stream ownership.** `stop()` tears down the AudioContext but does NOT stop the
+ *   `MediaStreamTrack`s passed in — callers own those.
+ */
+export class DualChannelCapture {
+  private readonly params: Required<
+    Omit<DualChannelCaptureParams, "targetSampleRate">
+  > & { targetSampleRate: number };
+  private errorListener?: ErrorListener;
+  private context?: AudioContext;
+  private micSource?: MediaStreamAudioSourceNode;
+  private sysSource?: MediaStreamAudioSourceNode;
+  private micEncoder?: AudioWorkletNode;
+  private sysEncoder?: AudioWorkletNode;
+  private running = false;
+
+  constructor(params: DualChannelCaptureParams) {
+    if (typeof globalThis.AudioContext === "undefined") {
+      throw new BrowserOnlyError();
+    }
+    this.params = {
+      micStream: params.micStream,
+      systemStream: params.systemStream,
+      transcriber: params.transcriber,
+      targetSampleRate: params.targetSampleRate ?? DEFAULT_TARGET_RATE,
+    };
+  }
+
+  on(event: "error", listener: ErrorListener): void {
+    if (event === "error") this.errorListener = listener;
+  }
+
+  /**
+   * Wire the capture pipeline and start pumping tagged PCM into the transcriber.
+   * The transcriber must already be connected. Returns once the worklet is
+   * registered and the audio graph is live.
+   */
+  async start(): Promise<void> {
+    if (this.running) {
+      throw new Error("DualChannelCapture already started");
+    }
+    this.context = new AudioContext();
+
+    const blob = new Blob([pcm16EncoderWorkletSource], {
+      type: "application/javascript",
+    });
+    const url = URL.createObjectURL(blob);
+    try {
+      await this.context.audioWorklet.addModule(url);
+    } finally {
+      URL.revokeObjectURL(url);
+    }
+
+    this.micSource = this.context.createMediaStreamSource(
+      this.params.micStream,
+    );
+    this.sysSource = this.context.createMediaStreamSource(
+      this.params.systemStream,
+    );
+
+    this.micEncoder = this.makeEncoder(MIC_CHANNEL);
+    this.sysEncoder = this.makeEncoder(SYSTEM_CHANNEL);
+    this.micSource.connect(this.micEncoder);
+    this.sysSource.connect(this.sysEncoder);
+
+    this.running = true;
+  }
+
+  private makeEncoder(channel: string): AudioWorkletNode {
+    const node = new AudioWorkletNode(
+      this.context!,
+      PCM16_ENCODER_PROCESSOR_NAME,
+      {
+        numberOfInputs: 1,
+        numberOfOutputs: 0,
+        channelCount: 1,
+        channelCountMode: "explicit",
+        channelInterpretation: "speakers",
+        processorOptions: {
+          targetRate: this.params.targetSampleRate,
+          chunkMs: DEFAULT_CHUNK_MS,
+        },
+      },
+    );
+    node.port.onmessage = (e: MessageEvent<Pcm16EncoderMessage>) => {
+      try {
+        this.params.transcriber.sendAudio(e.data.pcm, { channel });
+      } catch (err) {
+        this.errorListener?.(err as Error);
+      }
+    };
+    return node;
+  }
+
+  /**
+   * Tear down internal nodes and close the AudioContext. Does NOT stop the
+   * caller-provided MediaStream tracks — they remain available for preview UI,
+   * recording, etc. Idempotent.
+   */
+  async stop(): Promise<void> {
+    if (!this.running) return;
+    this.running = false;
+
+    try {
+      this.micEncoder?.port.close();
+      this.sysEncoder?.port.close();
+      this.micEncoder?.disconnect();
+      this.sysEncoder?.disconnect();
+      this.micSource?.disconnect();
+      this.sysSource?.disconnect();
+    } catch {
+      // Disconnecting already-disconnected nodes throws in some browsers; ignore.
+    }
+
+    if (this.context && this.context.state !== "closed") {
+      await this.context.close();
+    }
+
+    this.context = undefined;
+    this.micSource = undefined;
+    this.sysSource = undefined;
+    this.micEncoder = undefined;
+    this.sysEncoder = undefined;
+  }
+}

package/src/services/streaming/browser/worklets/pcm16-encoder.ts

@@ -0,0 +1,70 @@
+/**
+ * AudioWorklet processor that ingests mono Float32 audio at the AudioContext's
+ * native sample rate, resamples to `targetRate` (linear interpolation, stateful
+ * across `process()` calls), packs to little-endian Int16 PCM, and posts
+ * fixed-size chunks via `port.postMessage` with a running `samplesSent` counter.
+ *
+ * `samplesSent` is in **target-rate samples**, so the main thread can derive a
+ * stream-relative timestamp = `samplesSent / targetRate * 1000` (ms) — the same
+ * frame AAI uses for `StreamingWord.start` / `.end`.
+ *
+ * Defined as a string so it can be registered via a Blob URL — the SDK ships as
+ * a single ESM file, so a separate `.js` worklet asset isn't viable.
+ */
+export const pcm16EncoderWorkletSource = `
+class Pcm16EncoderProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    const opts = (options && options.processorOptions) || {};
+    this.targetRate = opts.targetRate || 16000;
+    this.chunkMs = opts.chunkMs || 50;
+    this.ratio = sampleRate / this.targetRate;
+    this.chunkSize = Math.round(this.targetRate * this.chunkMs / 1000);
+    this.buffer = new Int16Array(this.chunkSize);
+    this.bufferIdx = 0;
+    this.samplesSent = 0;
+    this.lastSample = 0;
+    this.fractional = 0;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || input.length === 0 || !input[0] || input[0].length === 0) {
+      return true;
+    }
+    const mono = input[0];
+    let pos = this.fractional;
+    while (pos < mono.length) {
+      const i = Math.floor(pos);
+      const frac = pos - i;
+      const a = i === 0 ? this.lastSample : mono[i - 1];
+      const b = mono[i];
+      const sample = a + (b - a) * frac;
+      const clamped = sample < -1 ? -1 : sample > 1 ? 1 : sample;
+      this.buffer[this.bufferIdx++] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
+      if (this.bufferIdx === this.chunkSize) {
+        const out = new Int16Array(this.chunkSize);
+        out.set(this.buffer);
+        this.samplesSent += this.chunkSize;
+        this.port.postMessage(
+          { pcm: out.buffer, samplesSent: this.samplesSent },
+          [out.buffer],
+        );
+        this.bufferIdx = 0;
+      }
+      pos += this.ratio;
+    }
+    this.lastSample = mono[mono.length - 1];
+    this.fractional = pos - mono.length;
+    return true;
+  }
+}
+registerProcessor("aai-pcm16-encoder", Pcm16EncoderProcessor);
+`;
+
+export const PCM16_ENCODER_PROCESSOR_NAME = "aai-pcm16-encoder";
+
+export type Pcm16EncoderMessage = {
+  pcm: ArrayBuffer;
+  samplesSent: number;
+};

package/src/services/streaming/energy-vad.ts

@@ -0,0 +1,75 @@
+import {
+  VadDetector,
+  VadDetectorResult,
+} from "../../types/streaming/dual-channel";
+
+export type EnergyVadParams = {
+  /** Threshold = noiseFloor * thresholdRatio. Default 3.0 (~ +9.5 dB above noise). */
+  thresholdRatio?: number;
+  /** EMA smoothing for the noise-floor estimate when frame is non-speech. Default 0.05. */
+  noiseFloorAlpha?: number;
+  /** Hangover in frames: stay "active" this many frames after the last speech frame. Default 10 (\~200 ms at 20 ms frames). */
+  hangoverFrames?: number;
+  /** Initial noise floor estimate. Default 1e-4. Adaptive after the first non-speech frame. */
+  initialNoiseFloor?: number;
+};
+
+/**
+ * Energy-based VAD with adaptive noise-floor tracking and hangover. Pure JS,
+ * no dependencies. Suitable for the "which physical channel is speaking" task
+ * because the channels are already physically separated at capture — the harder
+ * problem (speech vs. non-speech in the wild) is one a customer can swap in a
+ * DNN VAD for via the `createVad` parameter.
+ *
+ * Tuning notes:
+ * - thresholdRatio below 2 will treat anything above noise as speech (too sensitive).
+ * - thresholdRatio above 6 will miss quiet utterance onsets/offsets.
+ * - noiseFloorAlpha above 0.1 makes the floor track quickly (good for non-stationary
+ *   background) but risks slowly adapting *up* to a sustained low voice.
+ */
+export class EnergyVad implements VadDetector {
+  private readonly thresholdRatio: number;
+  private readonly noiseFloorAlpha: number;
+  private readonly hangoverFrames: number;
+  private readonly initialNoiseFloor: number;
+  private noiseFloor: number;
+  private hangoverRemaining = 0;
+
+  constructor(params: EnergyVadParams = {}) {
+    this.thresholdRatio = params.thresholdRatio ?? 3.0;
+    this.noiseFloorAlpha = params.noiseFloorAlpha ?? 0.05;
+    this.hangoverFrames = params.hangoverFrames ?? 10;
+    this.initialNoiseFloor = params.initialNoiseFloor ?? 1e-4;
+    this.noiseFloor = this.initialNoiseFloor;
+  }
+
+  process(frame: Float32Array): VadDetectorResult {
+    let sumSq = 0;
+    for (let i = 0; i < frame.length; i++) {
+      sumSq += frame[i] * frame[i];
+    }
+    const rms = frame.length > 0 ? Math.sqrt(sumSq / frame.length) : 0;
+
+    const threshold = this.noiseFloor * this.thresholdRatio;
+    let active = rms > threshold;
+
+    if (active) {
+      this.hangoverRemaining = this.hangoverFrames;
+    } else if (this.hangoverRemaining > 0) {
+      this.hangoverRemaining--;
+      active = true;
+      // While in hangover, do not update noise floor — RMS may still reflect tail energy.
+    } else {
+      this.noiseFloor =
+        this.noiseFloor * (1 - this.noiseFloorAlpha) +
+        rms * this.noiseFloorAlpha;
+    }
+
+    return { active, energy: rms };
+  }
+
+  reset(): void {
+    this.noiseFloor = this.initialNoiseFloor;
+    this.hangoverRemaining = 0;
+  }
+}

package/src/services/streaming/index.ts

@@ -1,2 +1,6 @@
 export * from "./factory";
 export * from "./service";
+export * from "./browser/dual-channel-capture";
+export * from "./energy-vad";
+export * from "./label-mapper";
+export * from "./resampler";

package/src/services/streaming/label-mapper.ts

@@ -0,0 +1,128 @@
+import { StreamingWord, TurnEvent } from "../../types/streaming";
+import { Channel, VadFrame } from "../../types/streaming/dual-channel";
+
+export type LabelMapperParams = {
+  /** Per-word energy ratio above which a channel is declared dominant. */
+  dominanceRatio: number;
+};
+
+/**
+ * Append-only ring buffer of VAD frames in stream-relative ms order.
+ * `pushFrame` is O(1) amortized; `framesInWindow` is O(n) over kept frames,
+ * which is fine for the per-word lookups we do (a 30 s window at 50 frames/s
+ * per channel × 2 channels = 3000 entries, scanned once per word).
+ *
+ * Runtime-agnostic — no DOM or Web Audio dependencies.
+ */
+export class VadTimeline {
+  private frames: VadFrame[] = [];
+  private head = 0;
+
+  constructor(private readonly windowMs: number) {}
+
+  pushFrame(frame: VadFrame): void {
+    this.frames.push(frame);
+    const cutoff = frame.ts - this.windowMs;
+    while (
+      this.head < this.frames.length &&
+      this.frames[this.head].ts < cutoff
+    ) {
+      this.head++;
+    }
+    if (this.head > 1024 && this.head * 2 > this.frames.length) {
+      this.frames = this.frames.slice(this.head);
+      this.head = 0;
+    }
+  }
+
+  framesInWindow(startMs: number, endMs: number): VadFrame[] {
+    const out: VadFrame[] = [];
+    for (let i = this.head; i < this.frames.length; i++) {
+      const f = this.frames[i];
+      if (f.ts < startMs) continue;
+      if (f.ts > endMs) break;
+      out.push(f);
+    }
+    return out;
+  }
+
+  clear(): void {
+    this.frames = [];
+    this.head = 0;
+  }
+}
+
+/**
+ * Sum per-channel active RMS over a window. Returns a Map from channel name
+ * to total score. Channels with zero score are omitted.
+ */
+function scoreChannels(frames: VadFrame[]): Map<string, number> {
+  const scores = new Map<string, number>();
+  for (const f of frames) {
+    if (!f.active) continue;
+    scores.set(f.channel, (scores.get(f.channel) ?? 0) + f.rms);
+  }
+  return scores;
+}
+
+/**
+ * Decide which channel was dominant during a word's `[start, end]` window.
+ *
+ * - If no channel has any active VAD energy → `"unknown"`.
+ * - If the top channel beats the runner-up by at least `dominanceRatio` → top channel.
+ * - Else: top channel wins on absolute score; exact ties → `"unknown"`.
+ */
+export function attributeWord(
+  word: StreamingWord,
+  timeline: VadTimeline,
+  params: LabelMapperParams,
+): Channel {
+  const scores = scoreChannels(timeline.framesInWindow(word.start, word.end));
+  if (scores.size === 0) return "unknown";
+  const sorted = [...scores.entries()].sort((a, b) => b[1] - a[1]);
+  if (sorted.length === 1) return sorted[0][0];
+  const [topName, topScore] = sorted[0];
+  const [runnerName, runnerScore] = sorted[1];
+  if (topScore >= params.dominanceRatio * runnerScore) return topName;
+  if (topScore > runnerScore) return topName;
+  if (runnerScore > topScore) return runnerName;
+  return "unknown";
+}
+
+/**
+ * Duration-weighted majority of word channels. `"unknown"` if there are no
+ * words, every word resolved to `"unknown"`, or two channels tie exactly.
+ */
+export function rollUpTurnChannel(words: StreamingWord[]): Channel {
+  const totals = new Map<string, number>();
+  for (const w of words) {
+    if (!w.channel || w.channel === "unknown") continue;
+    const dur = Math.max(0, w.end - w.start);
+    totals.set(w.channel, (totals.get(w.channel) ?? 0) + dur);
+  }
+  if (totals.size === 0) return "unknown";
+  const sorted = [...totals.entries()].sort((a, b) => b[1] - a[1]);
+  if (sorted.length === 1) return sorted[0][0];
+  const [topName, topMs] = sorted[0];
+  const [, runnerMs] = sorted[1];
+  if (topMs === runnerMs) return "unknown";
+  return topName;
+}
+
+/**
+ * Mutate `turn` in place: write `turn.words[i].channel` for every word and set
+ * `turn.channel` to the duration-weighted rollup.
+ *
+ * Returns `void` because the transcriber owns the `TurnEvent` ref and forwards
+ * the same object to the customer listener — no need to allocate a copy.
+ */
+export function attributeTurn(
+  turn: TurnEvent,
+  timeline: VadTimeline,
+  params: LabelMapperParams,
+): void {
+  for (const w of turn.words) {
+    w.channel = attributeWord(w, timeline, params);
+  }
+  turn.channel = rollUpTurnChannel(turn.words);
+}

package/src/services/streaming/resampler.ts

@@ -0,0 +1,69 @@
+/**
+ * Linear-interpolation resampler for streaming Float32 audio. Stateful across
+ * `process()` calls so chunk boundaries don't introduce phase discontinuities:
+ * the last input sample and a fractional read position are carried over.
+ *
+ * Linear interpolation is good enough for ASR ingest — the downstream
+ * StreamingTranscriber band-limits at the target rate anyway, and a polyphase
+ * filter would be overkill in the AudioWorklet hot path. If a customer needs
+ * higher quality they can supply their own VadDetector + bypass the encoder.
+ */
+export class LinearResampler {
+  private readonly ratio: number;
+  private lastSample = 0;
+  private fractional = 0;
+
+  constructor(
+    private readonly sourceRate: number,
+    private readonly targetRate: number,
+  ) {
+    if (sourceRate <= 0 || targetRate <= 0) {
+      throw new Error("sourceRate and targetRate must be positive");
+    }
+    this.ratio = sourceRate / targetRate;
+  }
+
+  process(input: Float32Array): Float32Array {
+    if (this.sourceRate === this.targetRate) {
+      return input;
+    }
+
+    // Worst-case output length; we'll slice to actual.
+    const out = new Float32Array(Math.ceil(input.length / this.ratio) + 1);
+    let outIdx = 0;
+    let pos = this.fractional;
+
+    while (pos < input.length) {
+      const i = Math.floor(pos);
+      const frac = pos - i;
+      const a = i === 0 ? this.lastSample : input[i - 1];
+      const b = input[i];
+      out[outIdx++] = a + (b - a) * frac;
+      pos += this.ratio;
+    }
+
+    this.lastSample = input[input.length - 1] ?? this.lastSample;
+    this.fractional = pos - input.length;
+    return out.subarray(0, outIdx);
+  }
+
+  reset(): void {
+    this.lastSample = 0;
+    this.fractional = 0;
+  }
+}
+
+/** Convert Float32 PCM (-1..1) to little-endian Int16 PCM. */
+export function float32ToPcm16(input: Float32Array): ArrayBuffer {
+  const out = new ArrayBuffer(input.length * 2);
+  const view = new DataView(out);
+  for (let i = 0; i < input.length; i++) {
+    const clamped = Math.max(-1, Math.min(1, input[i]));
+    view.setInt16(
+      i * 2,
+      clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff,
+      true,
+    );
+  }
+  return out;
+}