assemblyai 4.33.3 → 4.34.0

package/dist/services/index.d.ts

@@ -3,7 +3,7 @@ import { LemurService } from "./lemur";
 import { RealtimeTranscriber, RealtimeTranscriberFactory, RealtimeService, RealtimeServiceFactory } from "./realtime";
 import { TranscriptService } from "./transcripts";
 import { FileService } from "./files";
-import { StreamingTranscriber, StreamingTranscriberFactory } from "./streaming";
+import { StreamingTranscriber, StreamingTranscriberFactory, DualChannelCapture, EnergyVad, LinearResampler, VadTimeline, attributeTurn, attributeWord, rollUpTurnChannel, float32ToPcm16 } from "./streaming";
 declare class AssemblyAI {
     /**
      * The files service.
@@ -31,4 +31,4 @@ declare class AssemblyAI {
      */
     constructor(params: BaseServiceParams);
 }
-export { AssemblyAI, LemurService, RealtimeTranscriberFactory, RealtimeTranscriber, RealtimeServiceFactory, RealtimeService, TranscriptService, FileService, StreamingTranscriber, };
+export { AssemblyAI, LemurService, RealtimeTranscriberFactory, RealtimeTranscriber, RealtimeServiceFactory, RealtimeService, TranscriptService, FileService, StreamingTranscriber, DualChannelCapture, EnergyVad, LinearResampler, VadTimeline, attributeTurn, attributeWord, rollUpTurnChannel, float32ToPcm16, };

package/dist/services/streaming/browser/dual-channel-capture.d.ts

@@ -0,0 +1,66 @@
+import { StreamingTranscriber } from "../service";
+export { BrowserOnlyError } from "../../../types/streaming/dual-channel";
+type ErrorListener = (err: Error) => void;
+export type DualChannelCaptureParams = {
+    /** Microphone MediaStream. Caller should set `echoCancellation: true` at `getUserMedia` time. */
+    micStream: MediaStream;
+    /** System-audio MediaStream (e.g. `getDisplayMedia({ audio: true })`). */
+    systemStream: MediaStream;
+    /**
+     * The transcriber to push tagged PCM into. MUST be constructed with
+     * `channels: [{ name: "mic" }, { name: "system" }]` so the per-channel
+     * `sendAudio` calls succeed.
+     */
+    transcriber: StreamingTranscriber;
+    /**
+     * Target sample rate sent to the transcriber. Defaults to 16000. The
+     * AudioContext runs at the device's native rate; resampling happens inside
+     * the encoder worklet (forcing `AudioContext({ sampleRate })` is unreliable
+     * across browsers).
+     */
+    targetSampleRate?: number;
+};
+/**
+ * Browser-only adapter that pumps two `MediaStream`s into a `StreamingTranscriber`
+ * configured for dual-channel mode. Each `MediaStream` runs through its own
+ * `pcm16-encoder` AudioWorklet (resample to `targetSampleRate`, encode to Int16
+ * PCM); each PCM chunk is forwarded via `transcriber.sendAudio(pcm, { channel })`.
+ *
+ * All dual-channel orchestration (mixing, VAD, per-word attribution) lives inside
+ * `StreamingTranscriber` — this class is a pure I/O adapter. Non-browser runtimes
+ * can replicate its job by pushing tagged PCM into `transcriber.sendAudio` directly.
+ *
+ * Caller responsibilities:
+ * - **Echo cancellation** is set at `getUserMedia` time (`audio: { echoCancellation: true }`).
+ * - **System-audio capture** is platform-dependent. Chrome's `getDisplayMedia({ audio: true })`
+ *   captures tab audio (and on Windows, full system audio when sharing the whole screen).
+ *   macOS requires a virtual loopback driver (e.g. BlackHole) to expose system audio at all.
+ * - **Token auth.** Construct the transcriber with `token` — API-key auth is unsupported in browsers.
+ * - **Stream ownership.** `stop()` tears down the AudioContext but does NOT stop the
+ *   `MediaStreamTrack`s passed in — callers own those.
+ */
+export declare class DualChannelCapture {
+    private readonly params;
+    private errorListener?;
+    private context?;
+    private micSource?;
+    private sysSource?;
+    private micEncoder?;
+    private sysEncoder?;
+    private running;
+    constructor(params: DualChannelCaptureParams);
+    on(event: "error", listener: ErrorListener): void;
+    /**
+     * Wire the capture pipeline and start pumping tagged PCM into the transcriber.
+     * The transcriber must already be connected. Returns once the worklet is
+     * registered and the audio graph is live.
+     */
+    start(): Promise<void>;
+    private makeEncoder;
+    /**
+     * Tear down internal nodes and close the AudioContext. Does NOT stop the
+     * caller-provided MediaStream tracks — they remain available for preview UI,
+     * recording, etc. Idempotent.
+     */
+    stop(): Promise<void>;
+}

package/dist/services/streaming/browser/worklets/pcm16-encoder.d.ts

@@ -0,0 +1,19 @@
+/**
+ * AudioWorklet processor that ingests mono Float32 audio at the AudioContext's
+ * native sample rate, resamples to `targetRate` (linear interpolation, stateful
+ * across `process()` calls), packs to little-endian Int16 PCM, and posts
+ * fixed-size chunks via `port.postMessage` with a running `samplesSent` counter.
+ *
+ * `samplesSent` is in **target-rate samples**, so the main thread can derive a
+ * stream-relative timestamp = `samplesSent / targetRate * 1000` (ms) — the same
+ * frame AAI uses for `StreamingWord.start` / `.end`.
+ *
+ * Defined as a string so it can be registered via a Blob URL — the SDK ships as
+ * a single ESM file, so a separate `.js` worklet asset isn't viable.
+ */
+export declare const pcm16EncoderWorkletSource = "\nclass Pcm16EncoderProcessor extends AudioWorkletProcessor {\n  constructor(options) {\n    super();\n    const opts = (options && options.processorOptions) || {};\n    this.targetRate = opts.targetRate || 16000;\n    this.chunkMs = opts.chunkMs || 50;\n    this.ratio = sampleRate / this.targetRate;\n    this.chunkSize = Math.round(this.targetRate * this.chunkMs / 1000);\n    this.buffer = new Int16Array(this.chunkSize);\n    this.bufferIdx = 0;\n    this.samplesSent = 0;\n    this.lastSample = 0;\n    this.fractional = 0;\n  }\n\n  process(inputs) {\n    const input = inputs[0];\n    if (!input || input.length === 0 || !input[0] || input[0].length === 0) {\n      return true;\n    }\n    const mono = input[0];\n    let pos = this.fractional;\n    while (pos < mono.length) {\n      const i = Math.floor(pos);\n      const frac = pos - i;\n      const a = i === 0 ? this.lastSample : mono[i - 1];\n      const b = mono[i];\n      const sample = a + (b - a) * frac;\n      const clamped = sample < -1 ? -1 : sample > 1 ? 1 : sample;\n      this.buffer[this.bufferIdx++] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;\n      if (this.bufferIdx === this.chunkSize) {\n        const out = new Int16Array(this.chunkSize);\n        out.set(this.buffer);\n        this.samplesSent += this.chunkSize;\n        this.port.postMessage(\n          { pcm: out.buffer, samplesSent: this.samplesSent },\n          [out.buffer],\n        );\n        this.bufferIdx = 0;\n      }\n      pos += this.ratio;\n    }\n    this.lastSample = mono[mono.length - 1];\n    this.fractional = pos - mono.length;\n    return true;\n  }\n}\nregisterProcessor(\"aai-pcm16-encoder\", Pcm16EncoderProcessor);\n";
+export declare const PCM16_ENCODER_PROCESSOR_NAME = "aai-pcm16-encoder";
+export type Pcm16EncoderMessage = {
+    pcm: ArrayBuffer;
+    samplesSent: number;
+};

package/dist/services/streaming/energy-vad.d.ts

@@ -0,0 +1,35 @@
+import { VadDetector, VadDetectorResult } from "../../types/streaming/dual-channel";
+export type EnergyVadParams = {
+    /** Threshold = noiseFloor * thresholdRatio. Default 3.0 (~ +9.5 dB above noise). */
+    thresholdRatio?: number;
+    /** EMA smoothing for the noise-floor estimate when frame is non-speech. Default 0.05. */
+    noiseFloorAlpha?: number;
+    /** Hangover in frames: stay "active" this many frames after the last speech frame. Default 10 (\~200 ms at 20 ms frames). */
+    hangoverFrames?: number;
+    /** Initial noise floor estimate. Default 1e-4. Adaptive after the first non-speech frame. */
+    initialNoiseFloor?: number;
+};
+/**
+ * Energy-based VAD with adaptive noise-floor tracking and hangover. Pure JS,
+ * no dependencies. Suitable for the "which physical channel is speaking" task
+ * because the channels are already physically separated at capture — the harder
+ * problem (speech vs. non-speech in the wild) is one a customer can swap in a
+ * DNN VAD for via the `createVad` parameter.
+ *
+ * Tuning notes:
+ * - thresholdRatio below 2 will treat anything above noise as speech (too sensitive).
+ * - thresholdRatio above 6 will miss quiet utterance onsets/offsets.
+ * - noiseFloorAlpha above 0.1 makes the floor track quickly (good for non-stationary
+ *   background) but risks slowly adapting *up* to a sustained low voice.
+ */
+export declare class EnergyVad implements VadDetector {
+    private readonly thresholdRatio;
+    private readonly noiseFloorAlpha;
+    private readonly hangoverFrames;
+    private readonly initialNoiseFloor;
+    private noiseFloor;
+    private hangoverRemaining;
+    constructor(params?: EnergyVadParams);
+    process(frame: Float32Array): VadDetectorResult;
+    reset(): void;
+}

package/dist/services/streaming/index.d.ts

@@ -1,2 +1,6 @@
 export * from "./factory";
 export * from "./service";
+export * from "./browser/dual-channel-capture";
+export * from "./energy-vad";
+export * from "./label-mapper";
+export * from "./resampler";

package/dist/services/streaming/label-mapper.d.ts

@@ -0,0 +1,44 @@
+import { StreamingWord, TurnEvent } from "../../types/streaming";
+import { Channel, VadFrame } from "../../types/streaming/dual-channel";
+export type LabelMapperParams = {
+    /** Per-word energy ratio above which a channel is declared dominant. */
+    dominanceRatio: number;
+};
+/**
+ * Append-only ring buffer of VAD frames in stream-relative ms order.
+ * `pushFrame` is O(1) amortized; `framesInWindow` is O(n) over kept frames,
+ * which is fine for the per-word lookups we do (a 30 s window at 50 frames/s
+ * per channel × 2 channels = 3000 entries, scanned once per word).
+ *
+ * Runtime-agnostic — no DOM or Web Audio dependencies.
+ */
+export declare class VadTimeline {
+    private readonly windowMs;
+    private frames;
+    private head;
+    constructor(windowMs: number);
+    pushFrame(frame: VadFrame): void;
+    framesInWindow(startMs: number, endMs: number): VadFrame[];
+    clear(): void;
+}
+/**
+ * Decide which channel was dominant during a word's `[start, end]` window.
+ *
+ * - If no channel has any active VAD energy → `"unknown"`.
+ * - If the top channel beats the runner-up by at least `dominanceRatio` → top channel.
+ * - Else: top channel wins on absolute score; exact ties → `"unknown"`.
+ */
+export declare function attributeWord(word: StreamingWord, timeline: VadTimeline, params: LabelMapperParams): Channel;
+/**
+ * Duration-weighted majority of word channels. `"unknown"` if there are no
+ * words, every word resolved to `"unknown"`, or two channels tie exactly.
+ */
+export declare function rollUpTurnChannel(words: StreamingWord[]): Channel;
+/**
+ * Mutate `turn` in place: write `turn.words[i].channel` for every word and set
+ * `turn.channel` to the duration-weighted rollup.
+ *
+ * Returns `void` because the transcriber owns the `TurnEvent` ref and forwards
+ * the same object to the customer listener — no need to allocate a copy.
+ */
+export declare function attributeTurn(turn: TurnEvent, timeline: VadTimeline, params: LabelMapperParams): void;

package/dist/services/streaming/resampler.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Linear-interpolation resampler for streaming Float32 audio. Stateful across
+ * `process()` calls so chunk boundaries don't introduce phase discontinuities:
+ * the last input sample and a fractional read position are carried over.
+ *
+ * Linear interpolation is good enough for ASR ingest — the downstream
+ * StreamingTranscriber band-limits at the target rate anyway, and a polyphase
+ * filter would be overkill in the AudioWorklet hot path. If a customer needs
+ * higher quality they can supply their own VadDetector + bypass the encoder.
+ */
+export declare class LinearResampler {
+    private readonly sourceRate;
+    private readonly targetRate;
+    private readonly ratio;
+    private lastSample;
+    private fractional;
+    constructor(sourceRate: number, targetRate: number);
+    process(input: Float32Array): Float32Array;
+    reset(): void;
+}
+/** Convert Float32 PCM (-1..1) to little-endian Int16 PCM. */
+export declare function float32ToPcm16(input: Float32Array): ArrayBuffer;

package/dist/services/streaming/service.d.ts

@@ -1,4 +1,13 @@
 import { StreamingTranscriberParams, AudioData, BeginEvent, TurnEvent, LLMGatewayResponseEvent, StreamingUpdateConfiguration, WarningEvent } from "../..";
+import type { VadFrame } from "../../types/streaming/dual-channel";
+/**
+ * Options for `sendAudio`. In dual-channel mode (when `channels` is configured
+ * on the transcriber), `channel` is required and must match one of the declared
+ * channel names; in single-channel mode it is ignored.
+ */
+export type SendAudioOptions = {
+    channel?: string;
+};
 export declare class StreamingTranscriber {
     private apiKey?;
     private token?;
@@ -6,17 +15,76 @@ export declare class StreamingTranscriber {
     private socket?;
     private listeners;
     private sessionTerminatedResolve?;
+    private isDualChannel;
+    private channelNames?;
+    private channelBuffers?;
+    private channelSamplesReceived?;
+    private channelVadFloatBuffers?;
+    private channelVadBufferIdx?;
+    private channelVads?;
+    private timeline?;
+    private flushTimer?;
+    private attributionParams?;
+    private vadFrameSamples;
+    private minChunkSamples;
+    private maxChunkSamples;
+    private speakerHistory?;
     constructor(params: StreamingTranscriberParams);
     private connectionUrl;
     on(event: "open", listener: (event: BeginEvent) => void): void;
     on(event: "turn", listener: (event: TurnEvent) => void): void;
     on(event: "llmGatewayResponse", listener: (event: LLMGatewayResponseEvent) => void): void;
     on(event: "warning", listener: (event: WarningEvent) => void): void;
+    on(event: "vad", listener: (event: VadFrame) => void): void;
     on(event: "error", listener: (error: Error) => void): void;
     on(event: "close", listener: (code: number, reason: string) => void): void;
     connect(): Promise<BeginEvent>;
+    /**
+     * Returns a WritableStream that pumps PCM chunks into `sendAudio`. Single-channel
+     * only — in dual-channel mode use `sendAudio(pcm, { channel })` directly, since
+     * `WritableStream` has no place to carry a channel tag.
+     */
     stream(): WritableStream<AudioData>;
-    sendAudio(audio: AudioData): void;
+    /**
+     * Send PCM audio.
+     *
+     * In single-channel mode, `audio` is forwarded directly to the WebSocket and
+     * `options` is ignored.
+     *
+     * In dual-channel mode (when `channels` is configured), `options.channel` is
+     * REQUIRED and must match one of the declared channel names. Per-channel PCM is
+     * fed into that channel's VAD, accumulated into a per-channel ring buffer, and
+     * a scheduled flush (`channelAttribution.flushIntervalMs`, default 50ms) mixes
+     * the buffers into mono before sending to the WebSocket.
+     */
+    sendAudio(audio: AudioData, options?: SendAudioOptions): void;
+    private ingestChannelAudio;
+    private startFlushTimer;
+    private flushMix;
+    /**
+     * Fill in words whose per-word VAD attribution was `"unknown"` by looking
+     * at the dominant non-`"unknown"` channel among ±N neighbors in the same
+     * turn. Words with no non-`"unknown"` neighbors stay `"unknown"`. Confident
+     * per-word VAD decisions are never modified.
+     *
+     * Local temporal heuristic — ignores `speaker_label`, so it works even when
+     * AAI's diarization re-uses the same label for two physically distinct
+     * voices. Each resolved word gets `channelResolved: true` so downstream
+     * renderers can distinguish inferred channels from directly-measured ones.
+     */
+    private resolveUnknownChannelsByWindow;
+    /**
+     * Fill `"unknown"` words by looking up the speaker's session-wide channel
+     * evidence. For each `speaker_label`, sums active VAD frame RMS per channel
+     * across every word the speaker has uttered to date. A speaker is
+     * "resolvable" if their total evidence clears
+     * `speakerHistoryMinRmsEvidence` and their top channel exceeds the
+     * runner-up by `speakerHistoryDominanceRatio`.
+     *
+     * Only touches `"unknown"` words. Confident per-word VAD decisions are
+     * never modified. `speaker_label` is never modified.
+     */
+    private resolveUnknownChannelsBySpeakerHistory;
     /**
      * Update the streaming configuration mid-stream.
      * @param config - The configuration parameters to update