@framers/agentos-ext-streaming-tts-elevenlabs 0.2.0

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@framers/agentos-ext-streaming-tts-elevenlabs",
+  "version": "0.2.0",
+  "description": "Streaming TTS via ElevenLabs WebSocket API for AgentOS voice pipeline",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "SKILL.md",
+    "manifest.json"
+  ],
+  "peerDependencies": {
+    "@framers/agentos": "^0.1.0"
+  },
+  "dependencies": {
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@types/ws": "^8.5.13",
+    "typescript": "^5.5.0",
+    "vitest": "^1.6.0"
+  },
+  "license": "MIT",
+  "author": "Frame.dev",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/framersai/agentos-extensions.git",
+    "directory": "registry/curated/voice/streaming-tts-elevenlabs"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "vitest run"
+  }
+}

package/src/ElevenLabsStreamingTTS.ts

@@ -0,0 +1,100 @@
+/**
+ * @file ElevenLabsStreamingTTS.ts
+ * @description {@link IStreamingTTS}-compatible factory for ElevenLabs TTS sessions.
+ *
+ * {@link ElevenLabsStreamingTTS} is a thin factory that creates
+ * {@link ElevenLabsTTSSession} instances on demand.  It holds no mutable state
+ * other than an active-session reference count used to implement {@link isStreaming}.
+ *
+ * @module streaming-tts-elevenlabs/ElevenLabsStreamingTTS
+ */
+
+import { ElevenLabsTTSSession } from './ElevenLabsTTSSession.js';
+import type { ElevenLabsStreamingTTSConfig, StreamingTTSConfig } from './types.js';
+
+// ---------------------------------------------------------------------------
+// Main class
+// ---------------------------------------------------------------------------
+
+/**
+ * Factory for ElevenLabs-backed streaming TTS sessions.
+ *
+ * Instantiate once per agent and reuse across voice turns.  Each call to
+ * {@link startSession} creates an independent {@link ElevenLabsTTSSession} that
+ * owns its own WebSocket connection lifecycle.
+ *
+ * @example
+ * ```ts
+ * const tts = new ElevenLabsStreamingTTS({ apiKey: process.env.ELEVENLABS_API_KEY! });
+ * const session = await tts.startSession();
+ *
+ * session.on('audio_chunk', (chunk) => audioPlayer.enqueue(chunk.audio));
+ *
+ * llm.on('token', (tok) => session.pushTokens(tok));
+ * llm.on('end',   ()    => session.flush());
+ * ```
+ */
+export class ElevenLabsStreamingTTS {
+  /**
+   * Stable provider identifier used by the voice pipeline to select between
+   * registered TTS implementations.
+   */
+  readonly providerId = 'elevenlabs-streaming-tts';
+
+  /** `true` while at least one session is open and not yet closed. */
+  get isStreaming(): boolean {
+    return this._activeSessions > 0;
+  }
+
+  /** Count of sessions opened but not yet closed or cancelled. */
+  private _activeSessions = 0;
+
+  /**
+   * @param config - ElevenLabs provider configuration.  `apiKey` is required;
+   *   all other fields have sensible defaults.
+   */
+  constructor(private readonly config: ElevenLabsStreamingTTSConfig) {}
+
+  /**
+   * Open a new streaming TTS session backed by a fresh WebSocket connection.
+   *
+   * Provider-specific options can be forwarded via `config.providerOptions`
+   * under the following keys:
+   * - `voiceId`        — ElevenLabs voice ID (default Rachel).
+   * - `modelId`        — Model ID (default `'eleven_turbo_v2'`).
+   * - `stability`      — Voice stability 0.0–1.0 (default `0.5`).
+   * - `similarityBoost`— Similarity boost 0.0–1.0 (default `0.75`).
+   * - `style`          — Style exaggeration 0.0–1.0 (default `0.0`).
+   * - `useSpeakerBoost`— Speaker boost enabled (default `true`).
+   *
+   * @param config - Generic session config merged with provider defaults.
+   * @returns A ready-to-use {@link ElevenLabsTTSSession}.
+   */
+  async startSession(config?: StreamingTTSConfig): Promise<ElevenLabsTTSSession> {
+    const provOpts = (config?.providerOptions ?? {}) as Partial<ElevenLabsStreamingTTSConfig>;
+
+    // Session-level overrides take precedence over factory-level config.
+    const sessionProviderConfig: ElevenLabsStreamingTTSConfig = {
+      ...this.config,
+      voiceId:        provOpts.voiceId         ?? this.config.voiceId,
+      modelId:        provOpts.modelId         ?? this.config.modelId,
+      stability:      provOpts.stability       ?? this.config.stability,
+      similarityBoost:provOpts.similarityBoost ?? this.config.similarityBoost,
+      style:          provOpts.style           ?? this.config.style,
+      useSpeakerBoost:provOpts.useSpeakerBoost ?? this.config.useSpeakerBoost,
+    };
+
+    const session = new ElevenLabsTTSSession(sessionProviderConfig, config ?? {});
+
+    this._activeSessions++;
+
+    // Decrement the counter when the session terminates for any reason.
+    const onDone = (): void => {
+      this._activeSessions = Math.max(0, this._activeSessions - 1);
+    };
+    session.once('close',     onDone);
+    session.once('cancelled', onDone);
+
+    return session;
+  }
+}

package/src/ElevenLabsTTSSession.ts

@@ -0,0 +1,385 @@
+/**
+ * @file ElevenLabsTTSSession.ts
+ * @description Active TTS session backed by the ElevenLabs WebSocket Streaming API.
+ *
+ * {@link ElevenLabsTTSSession} implements the `StreamingTTSSession` interface
+ * (EventEmitter) and provides the following events:
+ *
+ * | Event                | Payload                    | Description                                          |
+ * |----------------------|----------------------------|------------------------------------------------------|
+ * | `'audio_chunk'`      | `EncodedAudioChunk`        | MP3 audio buffer from ElevenLabs, ready for playback |
+ * | `'utterance_complete'`| `{ text, durationMs }`    | ElevenLabs signalled end of audio generation         |
+ * | `'cancelled'`        | `{ remaining: string }`    | Session cancelled; remaining text not rendered       |
+ * | `'error'`            | `Error`                    | WebSocket or protocol error                          |
+ * | `'close'`            | —                          | Session fully terminated                             |
+ *
+ * ### Protocol Overview
+ *
+ * The ElevenLabs WebSocket streaming protocol works as follows:
+ *
+ * 1. Open WSS connection to the voice/model endpoint.
+ * 2. Send a BOS (Beginning-Of-Stream) message: `{ text: " ", voice_settings: {...}, xi_api_key }`.
+ * 3. Stream text tokens as `{ text: "..." }` JSON messages.
+ * 4. To generate audio for accumulated text, include `flush: true` in the text message.
+ * 5. To end the stream, send an EOS (End-Of-Stream) message: `{ text: "" }`.
+ * 6. The server returns binary messages (MP3 audio chunks) and JSON status messages.
+ * 7. When all audio has been generated the server sends `{ isFinal: true }`.
+ *
+ * @module streaming-tts-elevenlabs/ElevenLabsTTSSession
+ */
+
+import { EventEmitter } from 'node:events';
+import WebSocket from 'ws';
+import type { ElevenLabsStreamingTTSConfig, StreamingTTSConfig, EncodedAudioChunk } from './types.js';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** ElevenLabs WebSocket API base URL. */
+const ELEVENLABS_WS_BASE = 'wss://api.elevenlabs.io';
+
+/** Default voice ID — Rachel (a neutral, clear English voice). */
+const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM';
+
+/** Default model — ElevenLabs Turbo v2, optimized for low latency. */
+const DEFAULT_MODEL_ID = 'eleven_turbo_v2';
+
+/** MP3 output format at 44.1 kHz / 128 kbps. */
+const OUTPUT_FORMAT = 'mp3_44100_128';
+
+/** Sample rate that matches the OUTPUT_FORMAT specifier. */
+const SAMPLE_RATE_HZ = 44_100;
+
+/**
+ * Rough estimate of spoken word rate used to compute {@link EncodedAudioChunk.durationMs}.
+ * 150 words per minute.
+ */
+const WORDS_PER_MS = 150 / 60_000;
+
+/** Regex matching sentence-final punctuation. */
+const SENTENCE_BOUNDARY_RE = /[.?!]/;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Estimate the spoken duration in milliseconds for a piece of text.
+ *
+ * Uses a simple word-count heuristic (150 WPM).  Callers should treat the result
+ * as a hint rather than a precise measurement.
+ *
+ * @param text - The text to estimate duration for.
+ * @returns Estimated playback duration in milliseconds (minimum 200 ms).
+ */
+function estimateDurationMs(text: string): number {
+  const wordCount = text.trim().split(/\s+/).filter(Boolean).length;
+  return Math.max(200, Math.round(wordCount / WORDS_PER_MS));
+}
+
+// ---------------------------------------------------------------------------
+// Main class
+// ---------------------------------------------------------------------------
+
+/**
+ * Active streaming TTS session backed by the ElevenLabs WebSocket Streaming API.
+ *
+ * Construct via {@link ElevenLabsStreamingTTS.startSession} rather than directly.
+ *
+ * @example
+ * ```ts
+ * const session = new ElevenLabsTTSSession(
+ *   { apiKey: process.env.ELEVENLABS_API_KEY! },
+ *   {},
+ * );
+ *
+ * session.on('audio_chunk', (chunk) => audioPlayer.enqueue(chunk.audio));
+ * session.on('utterance_complete', ({ text, durationMs }) => {
+ *   console.log(`Synthesis complete ~${durationMs} ms`);
+ * });
+ *
+ * llm.on('token', (tok) => session.pushTokens(tok));
+ * llm.on('end',   ()    => session.flush());
+ * ```
+ */
+export class ElevenLabsTTSSession extends EventEmitter {
+  // -------------------------------------------------------------------------
+  // Resolved config
+  // -------------------------------------------------------------------------
+
+  /** ElevenLabs voice ID. */
+  private readonly voiceId: string;
+
+  /** ElevenLabs model ID. */
+  private readonly modelId: string;
+
+  /** Voice stability (0.0 – 1.0). */
+  private readonly stability: number;
+
+  /** Similarity boost (0.0 – 1.0). */
+  private readonly similarityBoost: number;
+
+  /** Style exaggeration (0.0 – 1.0). */
+  private readonly style: number;
+
+  /** Whether to enable speaker boost. */
+  private readonly useSpeakerBoost: boolean;
+
+  // -------------------------------------------------------------------------
+  // Internal state
+  // -------------------------------------------------------------------------
+
+  /** Live WebSocket connection to the ElevenLabs streaming endpoint. */
+  private readonly ws: WebSocket;
+
+  /**
+   * Text that has been pushed since the last sentence-boundary flush.
+   *
+   * Accumulated so that `cancel()` can report the remaining un-synthesised text.
+   */
+  private pendingText = '';
+
+  /** `true` after {@link close} or {@link cancel} is called. */
+  private closed = false;
+
+  /**
+   * Whether the BOS (beginning-of-stream) message has been sent.
+   *
+   * The BOS is sent once on `ws.open`; subsequent messages are plain text sends.
+   */
+  private bosAcknowledged = false;
+
+  /**
+   * Queue of text segments that need to be sent once the WebSocket opens.
+   *
+   * If `pushTokens` or `flush` is called before the socket is open, messages
+   * are queued here and drained on `ws.open`.
+   */
+  private readonly sendQueue: Array<object> = [];
+
+  // -------------------------------------------------------------------------
+  // Constructor
+  // -------------------------------------------------------------------------
+
+  /**
+   * @param providerConfig - ElevenLabs-specific configuration (apiKey required).
+   * @param _sessionConfig - Generic voice-pipeline session config (reserved for
+   *   future `language` forwarding).
+   */
+  constructor(
+    private readonly providerConfig: ElevenLabsStreamingTTSConfig,
+    _sessionConfig: StreamingTTSConfig,
+  ) {
+    super();
+
+    // Apply defaults.
+    this.voiceId        = providerConfig.voiceId        ?? DEFAULT_VOICE_ID;
+    this.modelId        = providerConfig.modelId        ?? DEFAULT_MODEL_ID;
+    this.stability      = providerConfig.stability      ?? 0.5;
+    this.similarityBoost= providerConfig.similarityBoost ?? 0.75;
+    this.style          = providerConfig.style          ?? 0.0;
+    this.useSpeakerBoost= providerConfig.useSpeakerBoost ?? true;
+
+    // Build the WebSocket endpoint URL.
+    const wsUrl =
+      `${ELEVENLABS_WS_BASE}/v1/text-to-speech/${this.voiceId}/stream-input` +
+      `?model_id=${this.modelId}&output_format=${OUTPUT_FORMAT}`;
+
+    this.ws = new WebSocket(wsUrl);
+
+    this._wireWebSocket();
+  }
+
+  // -------------------------------------------------------------------------
+  // StreamingTTSSession interface
+  // -------------------------------------------------------------------------
+
+  /**
+   * Push LLM output token(s) into the stream.
+   *
+   * Text is accumulated in an internal buffer.  When a sentence boundary
+   * (`.`, `?`, `!`) is detected the accumulated segment is sent to ElevenLabs
+   * with `flush: true` so the server generates audio immediately.  Mid-sentence
+   * tokens are sent with a trailing space continuation to avoid falling intonation
+   * on partial phrases.
+   *
+   * @param text - One or more LLM token characters to append.
+   */
+  pushTokens(text: string): void {
+    if (this.closed) return;
+
+    this.pendingText += text;
+
+    // Check for sentence-boundary punctuation.
+    const hasBoundary = SENTENCE_BOUNDARY_RE.test(text);
+
+    if (hasBoundary) {
+      // Send the accumulated text with flush: true so ElevenLabs generates audio.
+      const segment = this.pendingText;
+      this.pendingText = '';
+      this._send({ text: segment, flush: true });
+    } else {
+      // Send the token chunk without flush; include a space continuation to hint
+      // that the phrase is mid-sentence (prevents falling prosody on partial text).
+      this._send({ text });
+    }
+  }
+
+  /**
+   * Flush any remaining buffered text and signal EOS to ElevenLabs.
+   *
+   * Call this when the LLM stream has ended.  Sends an EOS (end-of-stream)
+   * message (`{ text: "" }`) which instructs ElevenLabs to finalise audio
+   * generation for all remaining buffered text and close the stream.
+   *
+   * @returns Promise that resolves when the EOS message is queued for sending.
+   */
+  async flush(): Promise<void> {
+    if (this.closed) return;
+
+    // If there is buffered text that has not been sent, send it with flush: true first.
+    if (this.pendingText.trim().length > 0) {
+      const segment = this.pendingText;
+      this.pendingText = '';
+      this._send({ text: segment, flush: true });
+    }
+
+    // Send EOS — empty string signals end of stream to ElevenLabs.
+    this._send({ text: '' });
+  }
+
+  /**
+   * Cancel the session immediately.
+   *
+   * Closes the WebSocket, discards any unsent text, and emits a `'cancelled'`
+   * event carrying the remaining un-synthesised text.
+   */
+  cancel(): void {
+    if (this.closed) return;
+    this.closed = true;
+
+    const remaining = this.pendingText;
+    this.pendingText = '';
+
+    this.ws.close();
+    this.emit('cancelled', { remaining });
+  }
+
+  /**
+   * Close the session cleanly.
+   *
+   * Sends a WebSocket close frame and emits a `'close'` event.
+   * Idempotent — safe to call multiple times.
+   */
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+
+    this.ws.close();
+    this.emit('close');
+  }
+
+  // -------------------------------------------------------------------------
+  // Internal — WebSocket wiring
+  // -------------------------------------------------------------------------
+
+  /**
+   * Attach event listeners to the WebSocket instance and handle the full
+   * message lifecycle.
+   */
+  private _wireWebSocket(): void {
+    this.ws.on('open', () => {
+      // Send BOS message immediately on connection open.
+      const bosMessage = {
+        text: ' ',
+        voice_settings: {
+          stability:        this.stability,
+          similarity_boost: this.similarityBoost,
+          style:            this.style,
+          use_speaker_boost: this.useSpeakerBoost,
+        },
+        xi_api_key: this.providerConfig.apiKey,
+      };
+
+      // Use the raw WS send to bypass the queue (BOS must be first).
+      this.ws.send(JSON.stringify(bosMessage));
+      this.bosAcknowledged = true;
+
+      // Drain any messages queued before the socket opened.
+      for (const queued of this.sendQueue) {
+        this.ws.send(JSON.stringify(queued));
+      }
+      this.sendQueue.length = 0;
+    });
+
+    this.ws.on('message', (data: WebSocket.RawData, isBinary: boolean) => {
+      if (this.closed) return;
+
+      if (isBinary) {
+        // Binary message: MP3 audio chunk.
+        const audio = Buffer.isBuffer(data)
+          ? data
+          : Buffer.from(data as ArrayBuffer);
+
+        const durationMs = estimateDurationMs(this.pendingText);
+
+        const chunk: EncodedAudioChunk = {
+          audio,
+          format:     'mp3',
+          sampleRate: SAMPLE_RATE_HZ,
+          durationMs,
+          text:       '',
+        };
+
+        this.emit('audio_chunk', chunk);
+      } else {
+        // Text (JSON) message: parse for protocol signals.
+        try {
+          const msg = JSON.parse(data.toString()) as Record<string, unknown>;
+
+          if (msg['isFinal'] === true) {
+            // ElevenLabs has finished generating all audio for this stream.
+            const durationMs = estimateDurationMs(this.pendingText);
+            this.emit('utterance_complete', { text: this.pendingText, durationMs });
+          }
+
+          // Other JSON messages (e.g. alignment data) are silently ignored for now.
+        } catch {
+          // Malformed JSON from the server — emit a non-fatal error.
+          this.emit('error', new Error(`ElevenLabs: unexpected non-JSON text message: ${String(data)}`));
+        }
+      }
+    });
+
+    this.ws.on('error', (err: Error) => {
+      if (!this.closed) {
+        this.emit('error', err);
+      }
+    });
+
+    this.ws.on('close', () => {
+      if (!this.closed) {
+        this.closed = true;
+        this.emit('close');
+      }
+    });
+  }
+
+  /**
+   * Send a JSON message over the WebSocket.
+   *
+   * If the socket is not yet open, the message is queued and will be sent once
+   * the `open` event fires (after the BOS message is dispatched).
+   *
+   * @param msg - The message object to serialise and send.
+   */
+  private _send(msg: object): void {
+    if (this.bosAcknowledged && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify(msg));
+    } else {
+      // Queue for post-open delivery.
+      this.sendQueue.push(msg);
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,125 @@
+/**
+ * @file index.ts
+ * @description Pack factory for the ElevenLabs Streaming TTS extension pack.
+ *
+ * Exports the main {@link createElevenLabsStreamingTTS} factory function and the
+ * {@link createExtensionPack} bridge function conforming to the AgentOS manifest
+ * factory convention.
+ *
+ * ### Usage (direct)
+ * ```ts
+ * import { createElevenLabsStreamingTTS } from '@framers/agentos-ext-streaming-tts-elevenlabs';
+ *
+ * const tts = createElevenLabsStreamingTTS(process.env.ELEVENLABS_API_KEY!);
+ * const session = await tts.startSession();
+ *
+ * session.on('audio_chunk', (chunk) => audioPlayer.enqueue(chunk.audio));
+ * llm.on('token', (tok) => session.pushTokens(tok));
+ * llm.on('end',   ()    => session.flush());
+ * ```
+ *
+ * ### Usage (manifest-driven)
+ * ```json
+ * { "packs": [{ "module": "@framers/agentos-ext-streaming-tts-elevenlabs" }] }
+ * ```
+ *
+ * @module streaming-tts-elevenlabs
+ */
+
+import { ElevenLabsStreamingTTS } from './ElevenLabsStreamingTTS.js';
+import type { ElevenLabsStreamingTTSConfig } from './types.js';
+
+// ---------------------------------------------------------------------------
+// Local interface mirrors — avoids a hard runtime dep on @framers/agentos
+// ---------------------------------------------------------------------------
+
+/** Subset of ExtensionDescriptor required by this pack. */
+interface ExtensionDescriptor {
+  id: string;
+  kind: string;
+  payload: unknown;
+  enableByDefault?: boolean;
+  metadata?: Record<string, unknown>;
+}
+
+/** Subset of ExtensionPack required by this pack. */
+interface ExtensionPack {
+  id: string;
+  descriptors: ExtensionDescriptor[];
+}
+
+/** Subset of ExtensionPackContext required by this pack. */
+interface ExtensionPackContext {
+  getSecret?: (id: string) => string | undefined;
+  options?: Record<string, unknown>;
+}
+
+/** Kind constant matching packages/agentos/src/extensions/types.ts. */
+const EXTENSION_KIND_STREAMING_TTS = 'streaming-tts-provider';
+
+// ---------------------------------------------------------------------------
+// Factories
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a standalone {@link ElevenLabsStreamingTTS} instance.
+ *
+ * Use this when composing the provider programmatically outside of the
+ * AgentOS extension system.
+ *
+ * @param apiKey - ElevenLabs API key.
+ * @param options - Optional provider configuration overrides.
+ * @returns Configured {@link ElevenLabsStreamingTTS}.
+ */
+export function createElevenLabsStreamingTTS(
+  apiKey: string,
+  options?: Omit<ElevenLabsStreamingTTSConfig, 'apiKey'>,
+): ElevenLabsStreamingTTS {
+  return new ElevenLabsStreamingTTS({ apiKey, ...options });
+}
+
+/**
+ * AgentOS manifest factory function.
+ *
+ * Reads the `ELEVENLABS_API_KEY` secret from the context and returns an
+ * {@link ExtensionPack} containing a single `streaming-tts-provider`
+ * descriptor backed by {@link ElevenLabsStreamingTTS}.
+ *
+ * Provider-level defaults can be overridden via `context.options`:
+ * - `voiceId`        — ElevenLabs voice ID.
+ * - `modelId`        — Model ID (default `'eleven_turbo_v2'`).
+ * - `stability`      — Voice stability 0.0–1.0.
+ * - `similarityBoost`— Similarity boost 0.0–1.0.
+ * - `style`          — Style exaggeration 0.0–1.0.
+ * - `useSpeakerBoost`— Speaker boost.
+ *
+ * @param context - Pack context supplied by the extension manager.
+ * @returns A fully configured {@link ExtensionPack}.
+ */
+export function createExtensionPack(context: ExtensionPackContext): ExtensionPack {
+  const apiKey = context.getSecret?.('ELEVENLABS_API_KEY') ?? '';
+  const opts   = (context.options ?? {}) as Omit<ElevenLabsStreamingTTSConfig, 'apiKey'>;
+
+  const tts = new ElevenLabsStreamingTTS({ apiKey, ...opts });
+
+  return {
+    id: 'streaming-tts-elevenlabs',
+    descriptors: [
+      {
+        id:              'elevenlabs-streaming-tts',
+        kind:            EXTENSION_KIND_STREAMING_TTS,
+        payload:         tts,
+        enableByDefault: true,
+        metadata:        { providerId: 'elevenlabs-streaming-tts' },
+      },
+    ],
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Re-exports
+// ---------------------------------------------------------------------------
+
+export * from './types.js';
+export { ElevenLabsStreamingTTS }  from './ElevenLabsStreamingTTS.js';
+export { ElevenLabsTTSSession }    from './ElevenLabsTTSSession.js';