assemblyai 4.33.3 → 4.34.4

package/dist/types/streaming/dual-channel.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Physical input channel that a word/turn was attributed to.
+ * - A channel name declared in `StreamingTranscriberParams.channels` (e.g. `"mic"`, `"system"`).
+ * - `"unknown"`: no channel was clearly dominant during the word's time window (silent
+ *   or all channels evenly active under our threshold).
+ *
+ * This is independent of AssemblyAI's diarization `speaker_label` / `words[i].speaker`,
+ * which identifies voices by acoustic characteristics. A given speaker_label can map
+ * to any physical channel; the two dimensions can disagree.
+ */
+export type Channel = string | "unknown";
+/**
+ * Per-channel, per-frame VAD observation emitted by `StreamingTranscriber` when running
+ * in dual-channel mode. `ts` is stream-relative milliseconds, derived from the
+ * per-channel sample counter — the same reference frame as `StreamingWord.start` /
+ * `.end`, so per-word lookups need no conversion.
+ */
+export type VadFrame = {
+    ts: number;
+    channel: string;
+    active: boolean;
+    rms: number;
+};
+export type VadDetectorResult = {
+    active: boolean;
+    energy: number;
+};
+/**
+ * Pluggable per-channel voice-activity detector. The default `EnergyVad` is energy-based
+ * with an adaptive noise-floor threshold; callers can drop in a DNN-backed detector
+ * (e.g. Silero via `@ricky0123/vad-web`) for noisier environments.
+ *
+ * A separate `VadDetector` instance is held per channel; do not assume cross-channel
+ * state. Frames are fixed-size at the transcriber's target sample rate.
+ */
+export interface VadDetector {
+    process(frame: Float32Array): VadDetectorResult;
+    reset(): void;
+}
+/**
+ * Thrown when `DualChannelCapture` is constructed in a non-browser environment
+ * (no `globalThis.AudioContext`). The helper is intentionally surfaced from the
+ * main entrypoint so the import path is uniform across runtimes; the runtime
+ * guard moves to construction time.
+ */
+export declare class BrowserOnlyError extends Error {
+    constructor(message?: string);
+}

package/dist/types/streaming/index.d.ts

@@ -1,4 +1,63 @@
 import { AudioEncoding } from "..";
+import type { Channel, VadDetector, VadFrame } from "./dual-channel";
+export * from "./dual-channel";
+/**
+ * Per-channel attribution tuning for dual-channel mode. All fields optional;
+ * ignored when `StreamingTranscriberParams.channels` is not set.
+ */
+export type ChannelAttributionParams = {
+    /** Energy ratio above which a channel is declared dominant for a word. Default 4. */
+    dominanceRatio?: number;
+    /** Rolling VAD timeline window in ms. Default 30_000. */
+    timelineWindowMs?: number;
+    /**
+     * Factory for the per-channel VAD detector. Called once per declared channel
+     * at transcriber construction time. The channel name is passed so factories
+     * that wrap higher-level VAD libraries (which manage their own audio source)
+     * can map each `VadDetector` instance to its corresponding channel.
+     */
+    createVad?: (channelName: string) => VadDetector;
+    /** Mix flush interval in ms — how often per-channel buffers are summed and sent. Default 50. */
+    flushIntervalMs?: number;
+    /**
+     * Strategy used to fill words whose per-word VAD attribution resolved to
+     * `"unknown"`. Confident per-word VAD decisions (`"mic"` / `"system"`) are
+     * never modified by any strategy.
+     *
+     * - `"window"` (default): look at the dominant non-`"unknown"` channel
+     *   among ±`resolutionWindowWords` neighboring words in the same turn.
+     *   Ignores `speaker_label`, so it works even when AAI re-uses a label for
+     *   two physically distinct voices.
+     * - `"speaker-history"`: accumulate per-`speaker_label` per-channel active
+     *   VAD energy across the session, then fill `"unknown"` words with the
+     *   speaker's dominant channel when it clears
+     *   `speakerHistoryMinRmsEvidence` and beats runner-up by
+     *   `speakerHistoryDominanceRatio`. Robust for stable speaker labels but
+     *   does nothing when a speaker has split evidence.
+     * - `"none"`: disable resolution; `"unknown"` words remain `"unknown"` in
+     *   the output.
+     */
+    resolveUnknownChannelsMethod?: "none" | "window" | "speaker-history";
+    /**
+     * Half-window (in words) on each side of an `"unknown"` word for the
+     * `"window"` method. Default 2 — so the full window is up to 5 words
+     * (2 before + the unknown + 2 after).
+     */
+    resolutionWindowWords?: number;
+    /**
+     * Minimum cumulative active-RMS evidence (sum across all the speaker's
+     * frames to date) before a speaker can be resolved via the
+     * `"speaker-history"` method. Default 0.5 — roughly a few seconds of
+     * sustained speech.
+     */
+    speakerHistoryMinRmsEvidence?: number;
+    /**
+     * For the `"speaker-history"` method, the top channel's evidence must
+     * exceed the runner-up's by at least this factor for the speaker to be
+     * considered pinned to that channel. Default 3.
+     */
+    speakerHistoryDominanceRatio?: number;
+};
 export type LLMGatewayMessage = {
     role: string;
     content: string;
@@ -27,7 +86,8 @@ export type StreamingTranscriberParams = {
     keyterms?: string[];
     keytermsPrompt?: string[];
     prompt?: string;
-    speechModel: StreamingSpeechModel;
+    agentContext?: string;
+    speechModel?: StreamingSpeechModel;
     languageDetection?: boolean;
     domain?: StreamingDomain;
     inactivityTimeout?: number;
@@ -43,23 +103,56 @@ export type StreamingTranscriberParams = {
     redactPii?: boolean;
     redactPiiPolicies?: StreamingPiiPolicy[];
     redactPiiSub?: StreamingPiiSubstitution;
+    mode?: StreamingMode;
     llmGateway?: LLMGatewayConfig;
     webhookUrl?: string;
     webhookAuthHeaderName?: string;
     webhookAuthHeaderValue?: string;
+    /**
+     * Enable dual-channel (or N-channel) mode. Presence of `channels` switches the
+     * transcriber into channel-tagged mode: `sendAudio(audio, { channel })` is required,
+     * per-channel VAD runs on the raw PCM, the streams are mixed to mono before being
+     * sent to the server, and emitted `TurnEvent`s are enriched with `channel` and
+     * per-word `channel` attribution.
+     *
+     * Must contain exactly 2 entries with unique names. The names are echoed back in
+     * `TurnEvent.channel` / `words[i].channel`.
+     *
+     * **Acoustic-leak caveat.** Per-word channel attribution uses energy-based
+     * VAD on each channel. If your capture setup lets one channel's audio bleed
+     * into another at similar amplitude — typically system audio playing
+     * through speakers and being picked up by an open mic — attribution can
+     * misfire (mic-tagged words that were actually system). Transcription
+     * quality is unaffected; only the `channel` field is. To preserve
+     * attribution in speaker-leak setups, apply echo cancellation at capture
+     * before feeding audio to the SDK. In browsers, that's
+     * `getUserMedia({ audio: { echoCancellation: true } })`. On macOS native,
+     * `AVAudioEngine.setVoiceProcessingEnabled(true)` on the input node. If
+     * platform-level AEC isn't available, swap in a DNN VAD (e.g. Silero) via
+     * `channelAttribution.createVad`. See the dual-channel sample app's
+     * README for worked examples.
+     */
+    channels?: Array<{
+        name: string;
+    }>;
+    /** Tuning for dual-channel attribution. Ignored when `channels` is unset. */
+    channelAttribution?: ChannelAttributionParams;
 };
-export type StreamingEvents = "open" | "close" | "turn" | "speechStarted" | "llmGatewayResponse" | "warning" | "error";
+export type StreamingEvents = "open" | "close" | "turn" | "speechStarted" | "llmGatewayResponse" | "speakerRevision" | "warning" | "vad" | "error";
 export type StreamingListeners = {
     open?: (event: BeginEvent) => void;
     close?: (code: number, reason: string) => void;
     turn?: (event: TurnEvent) => void;
     speechStarted?: (event: SpeechStartedEvent) => void;
     llmGatewayResponse?: (event: LLMGatewayResponseEvent) => void;
+    speakerRevision?: (event: SpeakerRevisionEvent) => void;
     warning?: (event: WarningEvent) => void;
+    vad?: (event: VadFrame) => void;
     error?: (error: Error) => void;
 };
-export type StreamingSpeechModel = "universal-streaming-english" | "universal-streaming-multilingual" | "u3-rt-pro" | "whisper-rt" | "u3-pro";
+export type StreamingSpeechModel = "universal-streaming-english" | "universal-streaming-multilingual" | "u3-rt-pro" | "u3-rt-pro-beta-1" | "whisper-rt" | "u3-pro";
 export type StreamingDomain = "medical-v1";
+export type StreamingMode = "max_accuracy" | "min_latency" | "balanced";
 export type VoiceFocusModel = "near-field" | "far-field";
 export type StreamingPiiSubstitution = "hash" | "entity_name";
 export type StreamingPiiPolicy = "account_number" | "banking_information" | "blood_type" | "corporate_action" | "credit_card_cvv" | "credit_card_expiration" | "credit_card_number" | "date" | "date_interval" | "date_of_birth" | "day" | "drivers_license" | "drug" | "duration" | "effect" | "email_address" | "event" | "filename" | "financial_metric" | "gender" | "gender_sexuality" | "healthcare_number" | "injury" | "ip_address" | "language" | "location" | "location_address" | "location_address_street" | "location_city" | "location_coordinate" | "location_country" | "location_state" | "location_zip" | "marital_status" | "medical_code" | "medical_condition" | "medical_process" | "money_amount" | "month" | "nationality" | "number_sequence" | "occupation" | "organization" | "organization_id" | "organization_medical_facility" | "passport_number" | "password" | "person_age" | "person_name" | "phone_number" | "physical_attribute" | "political_affiliation" | "product" | "project" | "religion" | "sexuality" | "statistics" | "time" | "trend" | "url" | "us_social_security_number" | "username" | "vehicle_id" | "year" | "zodiac_sign";
@@ -91,6 +184,12 @@ export type TurnEvent = {
     language_code?: string;
     language_confidence?: number;
     speaker_label?: string;
+    /**
+     * Duration-weighted majority channel across `words[i].channel`. Populated only
+     * when the transcriber is configured with `channels`. Independent from
+     * `speaker_label`.
+     */
+    channel?: Channel;
 };
 export type StreamingWord = {
     start: number;
@@ -99,6 +198,20 @@ export type StreamingWord = {
     text: string;
     word_is_final: boolean;
     speaker?: string;
+    /**
+     * Physical input channel attributed by client-side VAD during this word's
+     * time window. Populated only when the transcriber is configured with
+     * `channels`. Independent from `speaker`.
+     */
+    channel?: Channel;
+    /**
+     * True if `channel` was filled in by `channelAttribution.resolveUnknownChannelsMethod`
+     * rather than by the per-word VAD. Only set on words whose per-word VAD
+     * attribution was `"unknown"` and whose resolution method produced a
+     * confident channel. Useful for debugging or rendering an indicator that a
+     * word's channel came from context, not direct VAD evidence.
+     */
+    channelResolved?: boolean;
 };
 export type TerminationEvent = {
     type: "Termination";
@@ -121,6 +234,7 @@ export type StreamingUpdateConfiguration = {
     format_turns?: boolean;
     keyterms_prompt?: string[];
     prompt?: string;
+    agent_context?: string;
     filter_profanity?: boolean;
     interruption_delay?: number;
     turn_left_pad_ms?: number;
@@ -144,5 +258,27 @@ export type LLMGatewayResponseEvent = {
     transcript: string;
     data: unknown;
 };
-export type StreamingEventMessage = BeginEvent | TurnEvent | SpeechStartedEvent | TerminationEvent | LLMGatewayResponseEvent | ErrorEvent | WarningEvent;
+/**
+ * A single earlier Turn whose speaker labels were revised by reclustering.
+ * Match by `turn_order` against the original Turn; replace its per-word
+ * `speaker` assignments (and the turn-level `speaker_label`) with these. Text
+ * and word timestamps are unchanged from the original Turn.
+ */
+export type SpeakerRevisionItem = {
+    turn_order: number;
+    speaker_label?: string;
+    words: StreamingWord[];
+};
+/**
+ * Server-side correction to previously-emitted Turns' speaker labels.
+ * Diarization-only (emitted only when `speakerLabels` is enabled). Sent once
+ * per offline-recluster resolve; `revisions` carries one entry per earlier
+ * Turn whose label actually changed (unchanged turns are omitted). Apply each
+ * entry by matching its `turn_order`.
+ */
+export type SpeakerRevisionEvent = {
+    type: "SpeakerRevision";
+    revisions: SpeakerRevisionItem[];
+};
+export type StreamingEventMessage = BeginEvent | TurnEvent | SpeechStartedEvent | TerminationEvent | LLMGatewayResponseEvent | SpeakerRevisionEvent | ErrorEvent | WarningEvent;
 export type StreamingOperationMessage = StreamingUpdateConfiguration | StreamingForceEndpoint | StreamingTerminateSession;