@framers/agentos-ext-streaming-stt-deepgram 0.2.0

package/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Copyright (c) 2025 Framers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+

package/SKILL.md

@@ -0,0 +1,62 @@
+---
+name: streaming-stt-deepgram
+description: Real-time streaming speech-to-text via Deepgram WebSocket API
+category: voice
+---
+
+# Deepgram Streaming STT
+
+Real-time streaming speech-to-text using Deepgram's Nova-2 model via WebSocket.
+
+## Setup
+
+Set `DEEPGRAM_API_KEY` in your environment or agent secrets store.
+
+## Features
+
+- Sub-300 ms latency interim transcripts
+- Native speaker diarization (word-level `speaker` labels)
+- Automatic punctuation insertion
+- Word-level timestamps and per-word confidence scores
+- Auto-reconnect on connection drops (exponential back-off 100 ms → 5 s)
+- Keyword boosting via `providerOptions.keywords`
+
+## Configuration
+
+In `agent.config.json`:
+
+```json
+{
+  "voice": {
+    "stt": "deepgram"
+  }
+}
+```
+
+Provider-specific options via `providerOptions`:
+
+```json
+{
+  "voice": {
+    "stt": "deepgram",
+    "providerOptions": {
+      "model": "nova-2",
+      "diarize": true,
+      "keywords": ["AgentOS:2", "Deepgram:1.5"],
+      "endpointing": 300
+    }
+  }
+}
+```
+
+## Events
+
+| Event                  | Payload           | Description                                     |
+|------------------------|-------------------|-------------------------------------------------|
+| `transcript`           | `TranscriptEvent` | Every hypothesis (interim + final)              |
+| `interim_transcript`   | `TranscriptEvent` | Non-final hypothesis                            |
+| `final_transcript`     | `TranscriptEvent` | Stable, final hypothesis                        |
+| `speech_start`         | —                 | First non-empty word in an utterance            |
+| `speech_end`           | —                 | Deepgram `speech_final` flag raised             |
+| `error`                | `Error`           | Unrecoverable provider error                    |
+| `close`                | —                 | Session fully terminated                        |

package/dist/DeepgramDiarizationAdapter.d.ts

@@ -0,0 +1,78 @@
+/**
+ * @file DeepgramDiarizationAdapter.ts
+ * @description Utilities for extracting speaker labels from Deepgram word-level
+ * diarization results.
+ *
+ * Deepgram's `diarize: true` option annotates each word in the transcript with
+ * a zero-indexed integer `speaker` field.  This module converts that raw
+ * numeric label into the string format expected by {@link TranscriptWord.speaker}
+ * (`'Speaker_0'`, `'Speaker_1'`, …) and determines the majority speaker for
+ * an utterance.
+ *
+ * @module streaming-stt-deepgram/DeepgramDiarizationAdapter
+ */
+/**
+ * A single Deepgram word object as returned by the WebSocket API when
+ * `diarize: true` is passed in the connection query string.
+ *
+ * Only the fields used by this adapter are declared; the raw JSON may contain
+ * additional fields that are ignored.
+ */
+export interface DeepgramWord {
+    /** Recognised word token. */
+    word: string;
+    /** Start time relative to the stream start in seconds. */
+    start: number;
+    /** End time relative to the stream start in seconds. */
+    end: number;
+    /** Recognition confidence in [0, 1]. */
+    confidence: number;
+    /**
+     * Zero-indexed speaker index assigned by Deepgram's diarization model.
+     * Present only when the session was opened with `diarize=true`.
+     */
+    speaker?: number;
+    /** Punctuated variant of {@link word} when `punctuate=true`. */
+    punctuated_word?: string;
+}
+/**
+ * Extract the majority speaker label from an array of Deepgram word objects.
+ *
+ * The function tallies votes by `speaker` index and returns the label for the
+ * speaker with the most words.  Ties are broken by lowest index (i.e. the
+ * earliest-assigned speaker wins).
+ *
+ * Returns `undefined` when the array is empty or no word carries a `speaker`
+ * field (i.e. diarization was not requested for this session).
+ *
+ * @param words - Raw Deepgram word objects from `channel.alternatives[0].words`.
+ * @returns A string label such as `'Speaker_0'`, or `undefined`.
+ *
+ * @example
+ * ```ts
+ * const words = [
+ *   { word: 'Hello', speaker: 0, start: 0, end: 0.3, confidence: 0.99 },
+ *   { word: 'world', speaker: 1, start: 0.4, end: 0.7, confidence: 0.97 },
+ *   { word: 'today', speaker: 0, start: 0.8, end: 1.1, confidence: 0.95 },
+ * ];
+ * extractSpeakerFromWords(words); // → 'Speaker_0'
+ * ```
+ */
+export declare function extractSpeakerFromWords(words: DeepgramWord[]): string | undefined;
+/**
+ * Map a Deepgram word object to the canonical `TranscriptWord`-compatible shape.
+ *
+ * Seconds are converted to milliseconds to match the voice pipeline convention.
+ * The `word` field prefers the punctuated variant when available.
+ *
+ * @param dw - Raw Deepgram word object.
+ * @returns A plain object matching the {@link TranscriptWord} interface.
+ */
+export declare function mapDeepgramWord(dw: DeepgramWord): {
+    word: string;
+    start: number;
+    end: number;
+    confidence: number;
+    speaker?: string;
+};
+//# sourceMappingURL=DeepgramDiarizationAdapter.d.ts.map

package/dist/DeepgramDiarizationAdapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeepgramDiarizationAdapter.d.ts","sourceRoot":"","sources":["../src/DeepgramDiarizationAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,SAAS,CAyBjF;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAQA"}

package/dist/DeepgramDiarizationAdapter.js

@@ -0,0 +1,78 @@
+/**
+ * @file DeepgramDiarizationAdapter.ts
+ * @description Utilities for extracting speaker labels from Deepgram word-level
+ * diarization results.
+ *
+ * Deepgram's `diarize: true` option annotates each word in the transcript with
+ * a zero-indexed integer `speaker` field.  This module converts that raw
+ * numeric label into the string format expected by {@link TranscriptWord.speaker}
+ * (`'Speaker_0'`, `'Speaker_1'`, …) and determines the majority speaker for
+ * an utterance.
+ *
+ * @module streaming-stt-deepgram/DeepgramDiarizationAdapter
+ */
+/**
+ * Extract the majority speaker label from an array of Deepgram word objects.
+ *
+ * The function tallies votes by `speaker` index and returns the label for the
+ * speaker with the most words.  Ties are broken by lowest index (i.e. the
+ * earliest-assigned speaker wins).
+ *
+ * Returns `undefined` when the array is empty or no word carries a `speaker`
+ * field (i.e. diarization was not requested for this session).
+ *
+ * @param words - Raw Deepgram word objects from `channel.alternatives[0].words`.
+ * @returns A string label such as `'Speaker_0'`, or `undefined`.
+ *
+ * @example
+ * ```ts
+ * const words = [
+ *   { word: 'Hello', speaker: 0, start: 0, end: 0.3, confidence: 0.99 },
+ *   { word: 'world', speaker: 1, start: 0.4, end: 0.7, confidence: 0.97 },
+ *   { word: 'today', speaker: 0, start: 0.8, end: 1.1, confidence: 0.95 },
+ * ];
+ * extractSpeakerFromWords(words); // → 'Speaker_0'
+ * ```
+ */
+export function extractSpeakerFromWords(words) {
+    if (words.length === 0)
+        return undefined;
+    /** Vote counts keyed by speaker index. */
+    const votes = new Map();
+    for (const w of words) {
+        if (w.speaker === undefined)
+            continue;
+        votes.set(w.speaker, (votes.get(w.speaker) ?? 0) + 1);
+    }
+    if (votes.size === 0)
+        return undefined;
+    // Find the speaker index with the highest vote count; ties broken by lowest index.
+    let majorityIndex = -1;
+    let majorityCount = -1;
+    for (const [index, count] of votes) {
+        if (count > majorityCount || (count === majorityCount && index < majorityIndex)) {
+            majorityCount = count;
+            majorityIndex = index;
+        }
+    }
+    return majorityIndex >= 0 ? `Speaker_${majorityIndex}` : undefined;
+}
+/**
+ * Map a Deepgram word object to the canonical `TranscriptWord`-compatible shape.
+ *
+ * Seconds are converted to milliseconds to match the voice pipeline convention.
+ * The `word` field prefers the punctuated variant when available.
+ *
+ * @param dw - Raw Deepgram word object.
+ * @returns A plain object matching the {@link TranscriptWord} interface.
+ */
+export function mapDeepgramWord(dw) {
+    return {
+        word: dw.punctuated_word ?? dw.word,
+        start: Math.round(dw.start * 1000),
+        end: Math.round(dw.end * 1000),
+        confidence: dw.confidence,
+        speaker: dw.speaker !== undefined ? `Speaker_${dw.speaker}` : undefined,
+    };
+}
+//# sourceMappingURL=DeepgramDiarizationAdapter.js.map

package/dist/DeepgramDiarizationAdapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeepgramDiarizationAdapter.js","sourceRoot":"","sources":["../src/DeepgramDiarizationAdapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AA2BH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAqB;IAC3D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEzC,0CAA0C;IAC1C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;YAAE,SAAS;QACtC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEvC,mFAAmF;IACnF,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;IACvB,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;IAEvB,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,aAAa,IAAI,CAAC,KAAK,KAAK,aAAa,IAAI,KAAK,GAAG,aAAa,CAAC,EAAE,CAAC;YAChF,aAAa,GAAG,KAAK,CAAC;YACtB,aAAa,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,aAAa,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACrE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,EAAgB;IAO9C,OAAO;QACL,IAAI,EAAE,EAAE,CAAC,eAAe,IAAI,EAAE,CAAC,IAAI;QACnC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC;QAClC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC;QAC9B,UAAU,EAAE,EAAE,CAAC,UAAU;QACzB,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS;KACxE,CAAC;AACJ,CAAC"}

package/dist/DeepgramStreamSession.d.ts

@@ -0,0 +1,150 @@
+/**
+ * @file DeepgramStreamSession.ts
+ * @description Active streaming STT session backed by the Deepgram WebSocket API.
+ *
+ * {@link DeepgramStreamSession} implements the {@link StreamingSTTSession} interface
+ * (which extends `EventEmitter`) and adds Deepgram-specific convenience events:
+ *
+ * | Event                 | Payload              | Description                                           |
+ * |-----------------------|----------------------|-------------------------------------------------------|
+ * | `'transcript'`        | `TranscriptEvent`    | Every hypothesis (interim + final, per base interface)|
+ * | `'interim_transcript'`| `TranscriptEvent`    | Non-final hypothesis only                             |
+ * | `'final_transcript'`  | `TranscriptEvent`    | Final (stable) hypothesis                             |
+ * | `'speech_start'`      | —                    | First non-empty word received in this utterance       |
+ * | `'speech_end'`        | —                    | Deepgram `speech_final` flag fired                    |
+ * | `'error'`             | `Error`              | Unrecoverable provider error                          |
+ * | `'close'`             | —                    | Session fully terminated                              |
+ *
+ * ### Auto-reconnect
+ * On an unexpected WebSocket close (code ≠ 1000/1001), the session buffers
+ * incoming audio frames (up to 5 s worth at 16 kHz / 512 samples per frame)
+ * and reattempts the connection with exponential back-off starting at 100 ms,
+ * doubling each attempt up to a cap of 5 000 ms.
+ *
+ * @module streaming-stt-deepgram/DeepgramStreamSession
+ */
+import { EventEmitter } from 'node:events';
+import type { DeepgramStreamingConfig } from './types.js';
+/** Minimal AudioFrame shape — mirrors packages/agentos/src/voice-pipeline/types.ts */
+export interface AudioFrame {
+    samples: Float32Array;
+    sampleRate: number;
+    timestamp: number;
+    speakerHint?: string;
+}
+/** Minimal TranscriptWord shape. */
+export interface TranscriptWord {
+    word: string;
+    start: number;
+    end: number;
+    confidence: number;
+    speaker?: string;
+}
+/** Minimal TranscriptEvent shape. */
+export interface TranscriptEvent {
+    text: string;
+    confidence: number;
+    words: TranscriptWord[];
+    isFinal: boolean;
+    durationMs?: number;
+}
+/**
+ * Active streaming STT session backed by a Deepgram WebSocket connection.
+ *
+ * Construct via {@link DeepgramStreamingSTT.startSession} rather than directly.
+ *
+ * @example
+ * ```ts
+ * const session = new DeepgramStreamSession({ apiKey: process.env.DEEPGRAM_API_KEY! });
+ *
+ * session.on('interim_transcript', (evt) => process.stdout.write(`\r${evt.text}`));
+ * session.on('final_transcript',   (evt) => console.log(`FINAL: ${evt.text}`));
+ * session.on('speech_end',         ()    => console.log('utterance ended'));
+ *
+ * // Push audio frames from a microphone source:
+ * microphone.on('frame', (frame) => session.pushAudio(frame));
+ *
+ * // Signal end of utterance:
+ * await session.flush();
+ * session.close();
+ * ```
+ */
+export declare class DeepgramStreamSession extends EventEmitter {
+    /** Resolved configuration with all defaults applied. */
+    private readonly cfg;
+    /** Current WebSocket connection (replaced on reconnect). */
+    private ws;
+    /** `true` once the first non-empty transcript arrives in this utterance. */
+    private speechStarted;
+    /** `true` after {@link close} is called — prevents reconnect attempts. */
+    private closed;
+    /**
+     * Ring buffer of audio frames accumulated while the WebSocket is
+     * reconnecting.  Capped at {@link MAX_BUFFER_FRAMES} to bound memory.
+     */
+    private reconnectBuffer;
+    /** Current reconnect back-off delay in milliseconds. */
+    private backoffMs;
+    /** Active reconnect timer handle. */
+    private reconnectTimer;
+    /** `true` while the WebSocket is open and ready to accept binary data. */
+    private wsReady;
+    /**
+     * @param config - Deepgram streaming configuration.  `apiKey` is required;
+     *                 all other fields have sensible defaults.
+     */
+    constructor(config: DeepgramStreamingConfig);
+    /**
+     * Push a raw PCM audio frame into the recognition stream.
+     *
+     * Converts Float32 samples to Int16 linear PCM and transmits them as binary
+     * over the WebSocket.  Frames received while the socket is reconnecting are
+     * queued in {@link reconnectBuffer} (up to {@link MAX_BUFFER_FRAMES}).
+     *
+     * @param frame - Audio frame with normalised Float32 samples at any sample rate.
+     *                Frames are sent as-is; resample to 16 kHz before calling if
+     *                the source differs from the Deepgram session's `sample_rate`.
+     */
+    pushAudio(frame: AudioFrame): void;
+    /**
+     * Signal end-of-utterance to Deepgram.
+     *
+     * Sends the JSON `{"type":"CloseStream"}` control message, instructing
+     * Deepgram to flush any buffered audio and emit a final transcript before
+     * closing the connection from their side.
+     *
+     * @returns A promise that resolves immediately after the message is enqueued.
+     */
+    flush(): Promise<void>;
+    /**
+     * Immediately terminate the session without waiting for a final result.
+     *
+     * Cancels any pending reconnect timer, marks the session as closed, and
+     * terminates the underlying WebSocket.  After `close()` no further events
+     * are emitted.
+     */
+    close(): void;
+    /**
+     * Open (or reopen) the Deepgram WebSocket connection.
+     *
+     * Attaches `open`, `message`, `error`, and `close` handlers.  On successful
+     * open the reconnect buffer is drained in order.
+     */
+    private connect;
+    /**
+     * Schedule a reconnect attempt after the current {@link backoffMs} delay,
+     * then double the delay for the next attempt (capped at {@link BACKOFF_MAX_MS}).
+     */
+    private scheduleReconnect;
+    /**
+     * Process a parsed Deepgram WebSocket message and emit the appropriate events.
+     *
+     * Deepgram sends two primary result types:
+     * - `Results` — transcript hypotheses with `is_final` and `speech_final` flags.
+     * - `Metadata` — session-level metadata (ignored here).
+     *
+     * @param msg - Parsed Deepgram JSON message.
+     */
+    private handleMessage;
+}
+//# sourceMappingURL=DeepgramStreamSession.d.ts.map

package/dist/DeepgramStreamSession.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeepgramStreamSession.d.ts","sourceRoot":"","sources":["../src/DeepgramStreamSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAQ1D,sFAAsF;AACtF,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,YAAY,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,oCAAoC;AACpC,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AA6ED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,qBAAsB,SAAQ,YAAY;IAKrD,wDAAwD;IACxD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAMlB;IAEF,4DAA4D;IAC5D,OAAO,CAAC,EAAE,CAA0B;IAEpC,4EAA4E;IAC5E,OAAO,CAAC,aAAa,CAAS;IAE9B,0EAA0E;IAC1E,OAAO,CAAC,MAAM,CAAS;IAEvB;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAoB;IAE3C,wDAAwD;IACxD,OAAO,CAAC,SAAS,CAAsB;IAEvC,qCAAqC;IACrC,OAAO,CAAC,cAAc,CAA8C;IAEpE,0EAA0E;IAC1E,OAAO,CAAC,OAAO,CAAS;IAMxB;;;OAGG;gBACS,MAAM,EAAE,uBAAuB;IAoB3C;;;;;;;;;;OAUG;IACH,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAgBlC;;;;;;;;OAQG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B;;;;;;OAMG;IACH,KAAK,IAAI,IAAI;IAqBb;;;;;OAKG;IACH,OAAO,CAAC,OAAO;IA0Df;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;CAuDtB"}